freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into fix/quoted-charset-encoding

Browse Source
This commit is contained in:
Karim shoair
2026-06-07 17:58:06 +03:00
committed by GitHub
parent 6390c0af3d b3408c6f3c
commit 9352ccec3f
35 changed files with 150 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/cli/test_cli.py
+8 -1
View File
@@ -4,8 +4,9 @@ from unittest.mock import patch, MagicMock
import pytest_httpbin
from scrapling.parser import Selector
from scrapling import __version__
from scrapling.cli import (
shell, mcp, get, post, put, delete, fetch, stealthy_fetch
main, shell, mcp, get, post, put, delete, fetch, stealthy_fetch
)
@@ -32,6 +33,12 @@ class TestCLI:
def runner(self):
return CliRunner()
def test_version_flag(self, runner):
"""Test that the --version flag prints the Scrapling version and exits"""
result = runner.invoke(main, ['--version'])
assert result.exit_code == 0
assert result.output.strip() == f'Scrapling, version {__version__}'
def test_shell_command(self, runner):
"""Test shell command"""
with patch('scrapling.core.shell.CustomShell') as mock_shell:
tests/parser/test_adaptive.py
+26
View File
@@ -56,6 +56,32 @@ class TestParserAdaptive:
assert relocated[0].has_class("new-class")
assert relocated[0].css(".new-description")[0].text == "Description 1"
def test_relocation_auto_save_no_match_above_threshold(self):
"""Adaptive relocation with `auto_save=True` must not crash when no element
clears the `percentage` threshold (relocate() returns an empty list)."""
original_html = """
<div class="container">
<article class="product" id="target">
<h3>Widget</h3>
<p class="desc">A widget</p>
</article>
</div>
"""
# Unrelated structure so nothing can match a high threshold
changed_html = "<html><body><span>totally unrelated content</span></body></html>"
old_page = Selector(original_html, url="example.com", adaptive=True)
new_page = Selector(changed_html, url="example.com", adaptive=True)
old_page.css("#target", identifier="target", auto_save=True)
# Before the fix this raised `IndexError: list index out of range` because the
# guard checked `elements is not None` but relocate() returns [] (never None).
result = new_page.css(
"#target", identifier="target", adaptive=True, auto_save=True, percentage=95
)
assert list(result) == []
@pytest.mark.asyncio
async def test_element_relocation_async(self):
"""Test relocating element after structure change in async mode"""
tests/parser/test_find_similar_advanced.py
+31 -6
View File
@@ -2,6 +2,7 @@
Tests for Selector.find_similar() with non-default parameters.
Target file: tests/parser/test_general.py (append to TestSimilarElements class)
"""
import pytest
from scrapling import Selector
@@ -61,14 +62,10 @@ class TestFindSimilarAdvanced:
first = product_page.css("div.product")[0]
# Ignore both data-price and data-category → only class matters → all 3 divs match
ignore_all_data = first.find_similar(
similarity_threshold=0.2,
ignore_attributes=["data-price", "data-category"]
similarity_threshold=0.2, ignore_attributes=["data-price", "data-category"]
)
# Ignore nothing → data-category difference (fruit vs veggie) may reduce matches
ignore_nothing = first.find_similar(
similarity_threshold=0.9,
ignore_attributes=[]
)
ignore_nothing = first.find_similar(similarity_threshold=0.9, ignore_attributes=[])
assert len(ignore_all_data) >= len(ignore_nothing)
def test_find_similar_on_text_node_returns_empty(self, product_page):
@@ -76,3 +73,31 @@ class TestFindSimilarAdvanced:
text_node = product_page.css(".name::text")[0]
result = text_node.find_similar()
assert len(result) == 0
def test_find_similar_attribute_count_mismatch_scoring(self):
"""The similarity denominator uses max() of both attribute counts, so candidates
with fewer attributes don't get inflated scores and candidates with extra
attributes stay penalized."""
html = """
<html><body>
<div class="cards">
<div class="card" data-kind="primary" data-color="red" data-size="large">Alpha</div>
<div class="card">Beta</div>
<div class="card" data-kind="primary" data-color="red" data-size="large" data-id="x">Gamma</div>
<div class="card" data-kind="primary" data-color="red" data-size="large">Delta</div>
</div>
</body></html>
"""
page = Selector(html, adaptive=False)
first = page.css("div.card")[0] # Alpha
similar = first.find_similar(similarity_threshold=0.9, ignore_attributes=[])
texts = {el.text for el in similar}
# An exact attribute match must pass
assert "Delta" in texts
# Beta matches 1 of Alpha's 4 attributes; the old denominator counted candidate
# attributes only, inflating it to a perfect score (1.0 / 1)
assert "Beta" not in texts
# Gamma's extra attribute dilutes the score (4.0 / 5) - the intentional penalty
assert "Gamma" not in texts
tests/requirements.txt
+1 -1
View File
@@ -1,6 +1,6 @@
pytest>=2.8.0,<9
pytest-cov
playwright==1.59.0
playwright==1.60.0
werkzeug<3.0.0
pytest-httpbin==2.1.0
pytest-asyncio
tests/spiders/test_cache.py
+21
View File
@@ -49,6 +49,27 @@ class TestResponseCacheManager:
assert dict(restored.headers) == dict(original.headers)
assert dict(restored.request_headers) == dict(original.request_headers)
@pytest.mark.anyio
async def test_put_overwrites_existing_entry(self):
"""Re-caching the same fingerprint must replace the stored response.
Regression test for a Windows-only failure: ``Path.rename`` cannot
overwrite an existing destination on Windows (raising ``WinError 183``),
so the second ``put`` was caught by the error handler, the temp file was
removed, and ``get`` kept returning the stale body. ``Path.replace``
overwrites atomically on every platform.
"""
with tempfile.TemporaryDirectory() as tmpdir:
cache = ResponseCacheManager(tmpdir)
fp = b"\x05" * 20
await cache.put(fp, _make_response(body=b"<html>first</html>"), "GET")
await cache.put(fp, _make_response(body=b"<html>second</html>"), "GET")
restored = await cache.get(fp)
assert restored is not None
assert restored.body == b"<html>second</html>"
@pytest.mark.anyio
async def test_get_cache_miss(self):
with tempfile.TemporaryDirectory() as tmpdir: