Merge branch 'dev' into test/storage-core-coverage
This commit is contained in:
Karim shoair
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Tests for Selector.iterancestors() and Selector.find_ancestor() methods.
|
||||
Target file: tests/parser/test_general.py (append to TestElementNavigation class)
|
||||
"""
|
||||
import pytest
|
||||
from scrapling import Selector
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def nested_page():
|
||||
html = """
|
||||
<html><body>
|
||||
<div id="level1">
|
||||
<section id="level2" class="wrapper">
|
||||
<article id="level3" class="card">
|
||||
<p id="level4"><span id="target">deep text</span></p>
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
return Selector(html, adaptive=False)
|
||||
|
||||
|
||||
class TestAncestorNavigation:
|
||||
def test_iterancestors_returns_all_ancestors(self, nested_page):
|
||||
"""iterancestors() should yield every ancestor up to <html>"""
|
||||
target = nested_page.css("#target")[0]
|
||||
ancestor_tags = [a.tag for a in target.iterancestors()]
|
||||
# Expected order: p → article → section → div → body → html
|
||||
assert ancestor_tags[:4] == ["p", "article", "section", "div"]
|
||||
assert "body" in ancestor_tags
|
||||
assert "html" in ancestor_tags
|
||||
|
||||
def test_iterancestors_order_is_bottom_up(self, nested_page):
|
||||
"""iterancestors() should start from the immediate parent, not the root"""
|
||||
target = nested_page.css("#target")[0]
|
||||
first_ancestor = next(target.iterancestors())
|
||||
assert first_ancestor.attrib.get("id") == "level4"
|
||||
|
||||
def test_find_ancestor_returns_first_match(self, nested_page):
|
||||
"""find_ancestor() should return the closest ancestor matching the predicate"""
|
||||
target = nested_page.css("#target")[0]
|
||||
# Looking for the nearest ancestor with class "card"
|
||||
result = target.find_ancestor(lambda el: el.has_class("card"))
|
||||
assert result is not None
|
||||
assert result.attrib.get("id") == "level3"
|
||||
|
||||
def test_find_ancestor_returns_none_when_not_found(self, nested_page):
|
||||
"""find_ancestor() should return None if no ancestor matches"""
|
||||
target = nested_page.css("#target")[0]
|
||||
result = target.find_ancestor(lambda el: el.has_class("nonexistent-class"))
|
||||
assert result is None
|
||||
|
||||
def test_iterancestors_on_text_node_is_empty(self, nested_page):
|
||||
"""iterancestors() on a text node should yield nothing (not raise)"""
|
||||
text_node = nested_page.css("#target::text")[0]
|
||||
ancestors = list(text_node.iterancestors())
|
||||
assert ancestors == []
|
||||
|
||||
def test_find_ancestor_on_root_element_returns_none(self, nested_page):
|
||||
"""find_ancestor() on the root <html> element should return None gracefully"""
|
||||
# html element has no ancestors
|
||||
html_el = nested_page.css("html")[0]
|
||||
result = html_el.find_ancestor(lambda el: True)
|
||||
assert result is None
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
Tests for Selector.find_similar() with non-default parameters.
|
||||
Target file: tests/parser/test_general.py (append to TestSimilarElements class)
|
||||
"""
|
||||
import pytest
|
||||
from scrapling import Selector
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def product_page():
|
||||
html = """
|
||||
<html><body>
|
||||
<div class="product-list">
|
||||
<div class="product" data-category="fruit" data-price="10">
|
||||
<span class="name">Apple</span>
|
||||
</div>
|
||||
<div class="product" data-category="fruit" data-price="5">
|
||||
<span class="name">Banana</span>
|
||||
</div>
|
||||
<div class="product" data-category="veggie" data-price="3">
|
||||
<span class="name">Carrot</span>
|
||||
</div>
|
||||
<!-- Structurally similar but different tag — should NOT be found -->
|
||||
<section class="product" data-category="fruit" data-price="8">
|
||||
<span class="name">Grape</span>
|
||||
</section>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
return Selector(html, adaptive=False)
|
||||
|
||||
|
||||
class TestFindSimilarAdvanced:
|
||||
def test_find_similar_default_finds_same_tag_siblings(self, product_page):
|
||||
"""find_similar() with defaults should find div.product siblings, not the section"""
|
||||
first = product_page.css("div.product")[0]
|
||||
similar = first.find_similar()
|
||||
tags = [el.tag for el in similar]
|
||||
assert all(t == "div" for t in tags), "Should only return <div> elements"
|
||||
assert len(similar) == 2 # Banana and Carrot, not Grape (section)
|
||||
|
||||
def test_find_similar_high_threshold_filters_more(self, product_page):
|
||||
"""A higher similarity_threshold should return fewer (or equal) results"""
|
||||
first = product_page.css("div.product")[0]
|
||||
low_threshold = first.find_similar(similarity_threshold=0.1)
|
||||
high_threshold = first.find_similar(similarity_threshold=0.9)
|
||||
assert len(high_threshold) <= len(low_threshold)
|
||||
|
||||
def test_find_similar_match_text_excludes_different_text(self, product_page):
|
||||
"""match_text=True should factor in text content during similarity scoring"""
|
||||
first = product_page.css("div.product")[0] # Apple
|
||||
# With match_text=True and a high threshold, "Apple" vs "Banana"/"Carrot" text
|
||||
# should reduce similarity scores — result count may drop
|
||||
with_text = first.find_similar(similarity_threshold=0.8, match_text=True)
|
||||
without_text = first.find_similar(similarity_threshold=0.8, match_text=False)
|
||||
# match_text=True is stricter when text differs, so result should be <= without_text
|
||||
assert len(with_text) <= len(without_text)
|
||||
|
||||
def test_find_similar_ignore_attributes_affects_matching(self, product_page):
|
||||
"""Ignoring data-price should make more elements qualify as similar"""
|
||||
first = product_page.css("div.product")[0]
|
||||
# Ignore both data-price and data-category → only class matters → all 3 divs match
|
||||
ignore_all_data = first.find_similar(
|
||||
similarity_threshold=0.2,
|
||||
ignore_attributes=["data-price", "data-category"]
|
||||
)
|
||||
# Ignore nothing → data-category difference (fruit vs veggie) may reduce matches
|
||||
ignore_nothing = first.find_similar(
|
||||
similarity_threshold=0.9,
|
||||
ignore_attributes=[]
|
||||
)
|
||||
assert len(ignore_all_data) >= len(ignore_nothing)
|
||||
|
||||
def test_find_similar_on_text_node_returns_empty(self, product_page):
|
||||
"""find_similar() on a text node should return empty Selectors without raising"""
|
||||
text_node = product_page.css(".name::text")[0]
|
||||
result = text_node.find_similar()
|
||||
assert len(result) == 0
|
||||
@@ -250,6 +250,68 @@ class TestTextHandlerAdvanced:
|
||||
matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False)
|
||||
assert len(matches) == 1
|
||||
|
||||
def test_text_handler_regex_check_match(self):
|
||||
"""Test TextHandler.re() with check_match=True returns bool"""
|
||||
text = TextHandler("Price: $10.99")
|
||||
assert text.re(r"\$[\d.]+", check_match=True) is True
|
||||
assert text.re(r"no-match-pattern", check_match=True) is False
|
||||
|
||||
def test_text_handler_regex_replace_entities_false(self):
|
||||
"""Test TextHandler.re() with replace_entities=False preserves entities"""
|
||||
text = TextHandler("Hello & World")
|
||||
results = text.re(r"&", replace_entities=False)
|
||||
assert len(results) == 1
|
||||
assert results[0] == "&"
|
||||
|
||||
def test_text_handler_regex_with_groups(self):
|
||||
"""Test TextHandler.re() with capture groups flattens results"""
|
||||
text = TextHandler("name=Alice age=30 name=Bob age=25")
|
||||
results = text.re(r"name=(\w+) age=(\d+)")
|
||||
assert len(results) == 4
|
||||
assert "Alice" in results
|
||||
assert "30" in results
|
||||
|
||||
def test_text_handler_re_first_with_default(self):
|
||||
"""Test TextHandler.re_first() returns default when no match"""
|
||||
text = TextHandler("no numbers here")
|
||||
result = text.re_first(r"\d+", default="N/A")
|
||||
assert result == "N/A"
|
||||
|
||||
def test_text_handler_re_first_returns_first_match(self):
|
||||
"""Test TextHandler.re_first() returns first match"""
|
||||
text = TextHandler("a1 b2 c3")
|
||||
result = text.re_first(r"\d")
|
||||
assert result == "1"
|
||||
assert isinstance(result, TextHandler)
|
||||
|
||||
def test_text_handler_clean_with_entities(self):
|
||||
"""Test TextHandler.clean() with remove_entities=True"""
|
||||
text = TextHandler("Hello\t&\nWorld")
|
||||
cleaned = text.clean(remove_entities=True)
|
||||
assert "&" not in cleaned
|
||||
assert "&" in cleaned
|
||||
assert "\t" not in cleaned
|
||||
assert "\n" not in cleaned
|
||||
|
||||
def test_text_handler_clean_without_entities(self):
|
||||
"""Test TextHandler.clean() preserves entities by default"""
|
||||
text = TextHandler("Hello\t&\nWorld")
|
||||
cleaned = text.clean(remove_entities=False)
|
||||
assert "&" in cleaned
|
||||
|
||||
def test_text_handler_json_valid(self):
|
||||
"""Test TextHandler.json() with valid JSON"""
|
||||
text = TextHandler('{"key": "value", "num": 42}')
|
||||
data = text.json()
|
||||
assert data["key"] == "value"
|
||||
assert data["num"] == 42
|
||||
|
||||
def test_text_handler_json_invalid(self):
|
||||
"""Test TextHandler.json() raises on invalid JSON"""
|
||||
text = TextHandler("not json")
|
||||
with pytest.raises(Exception):
|
||||
text.json()
|
||||
|
||||
def test_text_handlers_operations(self):
|
||||
"""Test TextHandlers list operations"""
|
||||
handlers = TextHandlers([
|
||||
@@ -266,6 +328,37 @@ class TestTextHandlerAdvanced:
|
||||
assert handlers.get("default") == "First"
|
||||
assert TextHandlers([]).get("default") == "default"
|
||||
|
||||
def test_text_handlers_re(self):
|
||||
"""Test TextHandlers.re() flattens results across all elements"""
|
||||
handlers = TextHandlers([
|
||||
TextHandler("a1 b2"),
|
||||
TextHandler("c3 d4"),
|
||||
])
|
||||
results = handlers.re(r"[a-z]\d")
|
||||
assert isinstance(results, TextHandlers)
|
||||
assert len(results) == 4
|
||||
assert results[0] == "a1"
|
||||
assert results[3] == "d4"
|
||||
|
||||
def test_text_handlers_re_empty(self):
|
||||
"""Test TextHandlers.re() on empty list"""
|
||||
handlers = TextHandlers([])
|
||||
results = handlers.re(r"\d+")
|
||||
assert isinstance(results, TextHandlers)
|
||||
assert len(results) == 0
|
||||
|
||||
def test_text_handlers_re_no_matches(self):
|
||||
"""Test TextHandlers.re() when no element matches"""
|
||||
handlers = TextHandlers([TextHandler("abc"), TextHandler("def")])
|
||||
results = handlers.re(r"\d+")
|
||||
assert len(results) == 0
|
||||
|
||||
def test_text_handlers_extract(self):
|
||||
"""Test TextHandlers.extract() returns self"""
|
||||
handlers = TextHandlers([TextHandler("a"), TextHandler("b")])
|
||||
assert handlers.extract() is handlers
|
||||
assert handlers.getall() is handlers
|
||||
|
||||
|
||||
class TestSelectorsAdvanced:
|
||||
"""Test advanced Selectors functionality"""
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
Tests for Selectors.filter() method edge cases.
|
||||
Target file: tests/parser/test_parser_advanced.py (append to TestAdvancedSelectors class)
|
||||
"""
|
||||
import pytest
|
||||
from scrapling import Selector, Selectors
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def page():
|
||||
html = """
|
||||
<html><body>
|
||||
<ul>
|
||||
<li class="item" data-value="10">Apple</li>
|
||||
<li class="item" data-value="5">Banana</li>
|
||||
<li class="item" data-value="20">Cherry</li>
|
||||
<li class="item disabled" data-value="0">Durian</li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
return Selector(html, adaptive=False)
|
||||
|
||||
|
||||
class TestSelectorsFilter:
|
||||
def test_filter_basic(self, page):
|
||||
"""filter() should return only elements matching the predicate"""
|
||||
items = page.css("li.item")
|
||||
expensive = items.filter(lambda el: int(el.attrib.get("data-value", 0)) >= 10)
|
||||
assert len(expensive) == 2
|
||||
texts = expensive.getall()
|
||||
assert any("Apple" in t for t in texts)
|
||||
assert any("Cherry" in t for t in texts)
|
||||
|
||||
def test_filter_returns_empty_selectors_when_no_match(self, page):
|
||||
"""filter() should return an empty Selectors (not None/exception) when nothing matches"""
|
||||
items = page.css("li.item")
|
||||
result = items.filter(lambda el: int(el.attrib.get("data-value", 0)) > 9999)
|
||||
assert isinstance(result, Selectors)
|
||||
assert len(result) == 0
|
||||
assert result.first is None
|
||||
|
||||
def test_filter_all_pass(self, page):
|
||||
"""filter() with always-True predicate should return all elements"""
|
||||
items = page.css("li.item")
|
||||
result = items.filter(lambda el: True)
|
||||
assert len(result) == len(items)
|
||||
|
||||
def test_filter_chained(self, page):
|
||||
"""filter() should be chainable — apply two filters in sequence"""
|
||||
items = page.css("li.item")
|
||||
# First: value > 0, then: not disabled
|
||||
result = (
|
||||
items
|
||||
.filter(lambda el: int(el.attrib.get("data-value", 0)) > 0)
|
||||
.filter(lambda el: not el.has_class("disabled"))
|
||||
)
|
||||
assert len(result) == 3 # Apple, Banana, Cherry (Durian is disabled AND value=0)
|
||||
|
||||
def test_filter_on_empty_selectors(self):
|
||||
"""filter() on an already-empty Selectors should not raise"""
|
||||
empty = Selectors()
|
||||
result = empty.filter(lambda el: True)
|
||||
assert isinstance(result, Selectors)
|
||||
assert len(result) == 0
|
||||
@@ -613,8 +613,7 @@ class TestCheckpointMethods:
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_from_checkpoint_raises_when_disabled(self):
|
||||
engine = _make_engine() # no crawldir → checkpoint disabled
|
||||
with pytest.raises(RuntimeError):
|
||||
await engine._restore_from_checkpoint()
|
||||
assert (await engine._restore_from_checkpoint()) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
"""Tests for the SessionManager class."""
|
||||
|
||||
from unittest.mock import AsyncMock, PropertyMock
|
||||
|
||||
from scrapling.core._types import Any
|
||||
import pytest
|
||||
|
||||
from scrapling.spiders.session import SessionManager
|
||||
from scrapling.spiders.request import Request
|
||||
|
||||
|
||||
class MockSession: # type: ignore[type-arg]
|
||||
@@ -350,3 +353,57 @@ class TestSessionManagerIntegration:
|
||||
# After close - all inactive
|
||||
await manager.close()
|
||||
assert all(not s._is_alive for s in sessions)
|
||||
|
||||
|
||||
class TestSessionManagerFetch:
|
||||
"""Test SessionManager fetch behavior."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_preserves_request_method(self):
|
||||
"""Test that fetch does not mutate request._session_kwargs.
|
||||
|
||||
Previously, fetch() used pop("method") which removed the method
|
||||
key from the original request dict. This caused retried requests
|
||||
(via request.copy()) to lose their HTTP method and fall back to GET.
|
||||
"""
|
||||
from scrapling.engines.static import _ASyncSessionLogic
|
||||
from scrapling.fetchers import FetcherSession
|
||||
from scrapling.engines.toolbelt.custom import Response
|
||||
|
||||
mock_response = Response(
|
||||
url="https://example.com",
|
||||
content=b"ok",
|
||||
status=200,
|
||||
reason="OK",
|
||||
cookies={},
|
||||
headers={"content-type": "text/html"},
|
||||
request_headers={},
|
||||
)
|
||||
mock_response.meta = {}
|
||||
|
||||
mock_client = AsyncMock(spec=_ASyncSessionLogic)
|
||||
mock_client._make_request = AsyncMock(return_value=mock_response)
|
||||
|
||||
mock_session = AsyncMock(spec=FetcherSession)
|
||||
mock_session._client = mock_client
|
||||
mock_session._is_alive = True
|
||||
|
||||
manager = SessionManager()
|
||||
manager._sessions["default"] = mock_session
|
||||
manager._default_session_id = "default"
|
||||
manager._started = True
|
||||
|
||||
request = Request("https://example.com", method="POST", data={"key": "value"})
|
||||
|
||||
assert request._session_kwargs["method"] == "POST"
|
||||
|
||||
await manager.fetch(request)
|
||||
|
||||
# method must still be present after fetch
|
||||
assert "method" in request._session_kwargs
|
||||
assert request._session_kwargs["method"] == "POST"
|
||||
|
||||
# verify the correct method was passed to _make_request
|
||||
mock_client._make_request.assert_called_once()
|
||||
call_kwargs = mock_client._make_request.call_args
|
||||
assert call_kwargs.kwargs["method"] == "POST"
|
||||
|
||||
Reference in New Issue
Block a user