freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moving parser tests to a better structure

Browse Source 
and covering a bit more parts
This commit is contained in:
Karim shoair
2024-11-06 12:30:41 +02:00
parent 5548941cf1
commit d632bb14af
3 changed files with 57 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parser/__init__.py
View File
tests/parser/test_automatch.py
+56
View File
@@ -0,0 +1,56 @@
import unittest
from scrapling import Adaptor
class TestParserAutoMatch(unittest.TestCase):
def test_element_relocation(self):
"""Test relocating element after structure change"""
original_html = '''
<div class="container">
<section class="products">
<article class="product" id="p1">
<h3>Product 1</h3>
<p class="description">Description 1</p>
</article>
<article class="product" id="p2">
<h3>Product 2</h3>
<p class="description">Description 2</p>
</article>
</section>
</div>
'''
changed_html = '''
<div class="new-container">
<div class="product-wrapper">
<section class="products">
<article class="product new-class" data-id="p1">
<div class="product-info">
<h3>Product 1</h3>
<p class="new-description">Description 1</p>
</div>
</article>
<article class="product new-class" data-id="p2">
<div class="product-info">
<h3>Product 2</h3>
<p class="new-description">Description 2</p>
</div>
</article>
</section>
</div>
</div>
'''
old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
# 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors
_ = old_page.css('#p1, #p2', auto_save=True)[0]
relocated = new_page.css('#p1', auto_match=True)
self.assertIsNotNone(relocated)
self.assertEqual(relocated[0].attrib['data-id'], 'p1')
self.assertTrue(relocated[0].has_class('new-class'))
self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
tests/test_parser_functions.py → tests/parser/test_general.py
+1 -51
View File
@@ -112,7 +112,7 @@ class TestParser(unittest.TestCase):
def test_find_similar_elements(self):
"""Test Finding similar elements of an element"""
first_product = self.page.css('.product')[0]
first_product = self.page.css_first('.product')
similar_products = first_product.find_similar()
self.assertEqual(len(similar_products), 2)
@@ -265,56 +265,6 @@ class TestParser(unittest.TestCase):
self.assertEqual(attr_json, {'jsonable': 'data'})
self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
def test_element_relocation(self):
"""Test relocating element after structure change"""
original_html = '''
<div class="container">
<section class="products">
<article class="product" id="p1">
<h3>Product 1</h3>
<p class="description">Description 1</p>
</article>
<article class="product" id="p2">
<h3>Product 2</h3>
<p class="description">Description 2</p>
</article>
</section>
</div>
'''
changed_html = '''
<div class="new-container">
<div class="product-wrapper">
<section class="products">
<article class="product new-class" data-id="p1">
<div class="product-info">
<h3>Product 1</h3>
<p class="new-description">Description 1</p>
</div>
</article>
<article class="product new-class" data-id="p2">
<div class="product-info">
<h3>Product 2</h3>
<p class="new-description">Description 2</p>
</div>
</article>
</section>
</div>
</div>
'''
old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
# 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors
_ = old_page.css('#p1, #p2', auto_save=True)[0]
relocated = new_page.css('#p1', auto_match=True)
self.assertIsNotNone(relocated)
self.assertEqual(relocated[0].attrib['data-id'], 'p1')
self.assertTrue(relocated[0].has_class('new-class'))
self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
def test_performance(self):
"""Test parsing and selecting speed"""
import time