freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(parser): use max of both attribute counts in similarity scoring denominator

Browse Source 
Candidates with fewer attributes than the original got inflated scores because the denominator counted candidate attributes only, while the extra-attributes penalty direction worked as intended. Using `max()` on both counts fixes the inflation while keeping the penalty.

Closes #322
This commit is contained in:
Karim shoair
2026-06-07 15:51:10 +03:00
parent f7d1e338e0
commit 9c0c857245
2 changed files with 34 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+3 -1
View File
@@ -991,7 +991,9 @@ class Selector(SelectorsGeneration):
SequenceMatcher(None, v, candidate_attributes.get(k, "")).ratio() SequenceMatcher(None, v, candidate_attributes.get(k, "")).ratio()
for k, v in original_attributes.items() for k, v in original_attributes.items()
) )
checks += len(candidate_attributes) # Using `max` so candidates with extra attributes are penalized and candidates
# with fewer attributes don't get inflated scores from a smaller denominator
checks += max(len(original_attributes), len(candidate_attributes))
else: else:
if not candidate_attributes: if not candidate_attributes:
# Both don't have attributes, this must mean something # Both don't have attributes, this must mean something
tests/parser/test_find_similar_advanced.py
+31 -6
View File
@@ -2,6 +2,7 @@
Tests for Selector.find_similar() with non-default parameters. Tests for Selector.find_similar() with non-default parameters.
Target file: tests/parser/test_general.py (append to TestSimilarElements class) Target file: tests/parser/test_general.py (append to TestSimilarElements class)
""" """
import pytest import pytest
from scrapling import Selector from scrapling import Selector
@@ -61,14 +62,10 @@ class TestFindSimilarAdvanced:
first = product_page.css("div.product")[0] first = product_page.css("div.product")[0]
# Ignore both data-price and data-category → only class matters → all 3 divs match # Ignore both data-price and data-category → only class matters → all 3 divs match
ignore_all_data = first.find_similar( ignore_all_data = first.find_similar(
similarity_threshold=0.2, similarity_threshold=0.2, ignore_attributes=["data-price", "data-category"]
ignore_attributes=["data-price", "data-category"]
) )
# Ignore nothing → data-category difference (fruit vs veggie) may reduce matches # Ignore nothing → data-category difference (fruit vs veggie) may reduce matches
ignore_nothing = first.find_similar( ignore_nothing = first.find_similar(similarity_threshold=0.9, ignore_attributes=[])
similarity_threshold=0.9,
ignore_attributes=[]
)
assert len(ignore_all_data) >= len(ignore_nothing) assert len(ignore_all_data) >= len(ignore_nothing)
def test_find_similar_on_text_node_returns_empty(self, product_page): def test_find_similar_on_text_node_returns_empty(self, product_page):
@@ -76,3 +73,31 @@ class TestFindSimilarAdvanced:
text_node = product_page.css(".name::text")[0] text_node = product_page.css(".name::text")[0]
result = text_node.find_similar() result = text_node.find_similar()
assert len(result) == 0 assert len(result) == 0
def test_find_similar_attribute_count_mismatch_scoring(self):
"""The similarity denominator uses max() of both attribute counts, so candidates
with fewer attributes don't get inflated scores and candidates with extra
attributes stay penalized."""
html = """
<html><body>
<div class="cards">
<div class="card" data-kind="primary" data-color="red" data-size="large">Alpha</div>
<div class="card">Beta</div>
<div class="card" data-kind="primary" data-color="red" data-size="large" data-id="x">Gamma</div>
<div class="card" data-kind="primary" data-color="red" data-size="large">Delta</div>
</div>
</body></html>
"""
page = Selector(html, adaptive=False)
first = page.css("div.card")[0] # Alpha
similar = first.find_similar(similarity_threshold=0.9, ignore_attributes=[])
texts = {el.text for el in similar}
# An exact attribute match must pass
assert "Delta" in texts
# Beta matches 1 of Alpha's 4 attributes; the old denominator counted candidate
# attributes only, inflating it to a perfect score (1.0 / 1)
assert "Beta" not in texts
# Gamma's extra attribute dilutes the score (4.0 / 5) - the intentional penalty
assert "Gamma" not in texts