Product 1
-This is product 1
- $10.99 -In stock: 5
- Products
-
-
-
-
-
-
-
+@pytest.fixture
+def html_content():
+ return '''
+
+
+ Complex Web Page
+
+
+
+
+
+
+
+
+
+
+
Product 2
-This is product 2
- $20.99 -In stock: 3
- Product 3
-This is product 3
- $15.99 -Out of stock
- Products
+
+
+
+
+
+
+
+
+ Product 1
+This is product 1
+ $10.99 +In stock: 5
+ Product 2
+This is product 2
+ $20.99 +In stock: 3
+ Product 3
+This is product 3
+ $15.99 +Out of stock
+ Customer Reviews
+
+
-
+
+
+
+
+
+ '''
- def test_css_selector(self):
- """Test Selecting elements with complex CSS selectors"""
- elements = self.page.css('main #products .product-list article.product')
- self.assertEqual(len(elements), 3)
- in_stock_products = self.page.css(
+@pytest.fixture
+def page(html_content):
+ return Adaptor(html_content, auto_match=False)
+
+
+# CSS Selector Tests
+class TestCSSSelectors:
+ def test_basic_product_selection(self, page):
+ """Test selecting all product elements"""
+ elements = page.css('main #products .product-list article.product')
+ assert len(elements) == 3
+
+ def test_in_stock_product_selection(self, page):
+ """Test selecting in-stock products"""
+ in_stock_products = page.css(
'main #products .product-list article.product:not(:contains("Out of stock"))')
- self.assertEqual(len(in_stock_products), 2)
+ assert len(in_stock_products) == 2
- def test_xpath_selector(self):
- """Test Selecting elements with Complex XPath selectors"""
- reviews = self.page.xpath(
+
+# XPath Selector Tests
+class TestXPathSelectors:
+ def test_high_rating_reviews(self, page):
+ """Test selecting reviews with high ratings"""
+ reviews = page.xpath(
'//section[@id="reviews"]//div[contains(@class, "review") and @data-rating >= 4]'
)
- self.assertEqual(len(reviews), 2)
+ assert len(reviews) == 2
- high_priced_products = self.page.xpath(
+ def test_high_priced_products(self, page):
+ """Test selecting products above a certain price"""
+ high_priced_products = page.xpath(
'//article[contains(@class, "product")]'
'[number(translate(substring-after(.//span[@class="price"], "$"), ",", "")) > 15]'
)
- self.assertEqual(len(high_priced_products), 2)
+ assert len(high_priced_products) == 2
- def test_find_by_text(self):
- """Test Selecting elements with Text matching"""
- stock_info = self.page.find_by_regex(r'In stock: \d+', first_match=False)
- self.assertEqual(len(stock_info), 2)
- stock_info = self.page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
- self.assertEqual(stock_info.text, 'In stock: 5')
+# Text Matching Tests
+class TestTextMatching:
+ def test_regex_multiple_matches(self, page):
+ """Test finding multiple matches with regex"""
+ stock_info = page.find_by_regex(r'In stock: \d+', first_match=False)
+ assert len(stock_info) == 2
- stock_info = self.page.find_by_text(r'In stock:', partial=True, first_match=False)
- self.assertEqual(len(stock_info), 2)
+ def test_regex_first_match(self, page):
+ """Test finding the first match with regex"""
+ stock_info = page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
+ assert stock_info.text == 'In stock: 5'
- out_of_stock = self.page.find_by_text('Out of stock', partial=False, first_match=False)
- self.assertEqual(len(out_of_stock), 1)
+ def test_partial_text_match(self, page):
+ """Test finding elements with partial text match"""
+ stock_info = page.find_by_text(r'In stock:', partial=True, first_match=False)
+ assert len(stock_info) == 2
- def test_find_similar_elements(self):
- """Test Finding similar elements of an element"""
- first_product = self.page.css_first('.product')
+ def test_exact_text_match(self, page):
+ """Test finding elements with exact text match"""
+ out_of_stock = page.find_by_text('Out of stock', partial=False, first_match=False)
+ assert len(out_of_stock) == 1
+
+
+# Similar Elements Tests
+class TestSimilarElements:
+ def test_finding_similar_products(self, page):
+ """Test finding similar product elements"""
+ first_product = page.css_first('.product')
similar_products = first_product.find_similar()
- self.assertEqual(len(similar_products), 2)
+ assert len(similar_products) == 2
- first_review = self.page.find('div', class_='review')
+ def test_finding_similar_reviews(self, page):
+ """Test finding similar review elements with additional filtering"""
+ first_review = page.find('div', class_='review')
similar_high_rated_reviews = [
review
for review in first_review.find_similar()
if int(review.attrib.get('data-rating', 0)) >= 4
]
- self.assertEqual(len(similar_high_rated_reviews), 1)
+ assert len(similar_high_rated_reviews) == 1
- def test_expected_errors(self):
- """Test errors that should raised if it does"""
- with self.assertRaises(ValueError):
+
+# Error Handling Tests
+class TestErrorHandling:
+ def test_invalid_adaptor_initialization(self):
+ """Test various invalid Adaptor initializations"""
+ # No arguments
+ with pytest.raises(ValueError):
_ = Adaptor(auto_match=False)
- with self.assertRaises(TypeError):
+ # Invalid argument types
+ with pytest.raises(TypeError):
_ = Adaptor(root="ayo", auto_match=False)
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
_ = Adaptor(text=1, auto_match=False)
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
_ = Adaptor(body=1, auto_match=False)
- with self.assertRaises(ValueError):
- _ = Adaptor(self.html, storage=object, auto_match=True)
+ def test_invalid_storage(self, page, html_content):
+ """Test invalid storage parameter"""
+ with pytest.raises(ValueError):
+ _ = Adaptor(html_content, storage=object, auto_match=True)
- def test_pickleable(self):
- """Test that objects aren't pickleable"""
- table = self.page.css('.product-list')[0]
- with self.assertRaises(TypeError): # Adaptors
+ def test_bad_selectors(self, page):
+ """Test handling of invalid selectors"""
+ with pytest.raises((SelectorError, SelectorSyntaxError)):
+ page.css('4 ayo')
+
+ with pytest.raises((SelectorError, SelectorSyntaxError)):
+ page.xpath('4 ayo')
+
+
+# Pickling and Object Representation Tests
+class TestPicklingAndRepresentation:
+ def test_unpickleable_objects(self, page):
+ """Test that Adaptor objects cannot be pickled"""
+ table = page.css('.product-list')[0]
+ with pytest.raises(TypeError):
pickle.dumps(table)
- with self.assertRaises(TypeError): # Adaptor
+ with pytest.raises(TypeError):
pickle.dumps(table[0])
- def test_overridden(self):
- """Test overridden functions"""
- table = self.page.css('.product-list')[0]
- self.assertTrue(issubclass(type(table.__str__()), str))
- self.assertTrue(issubclass(type(table.__repr__()), str))
- self.assertTrue(issubclass(type(table.attrib.__str__()), str))
- self.assertTrue(issubclass(type(table.attrib.__repr__()), str))
+ def test_string_representations(self, page):
+ """Test custom string representations of objects"""
+ table = page.css('.product-list')[0]
+ assert issubclass(type(table.__str__()), str)
+ assert issubclass(type(table.__repr__()), str)
+ assert issubclass(type(table.attrib.__str__()), str)
+ assert issubclass(type(table.attrib.__repr__()), str)
- def test_bad_selector(self):
- """Test object can handle bad selector"""
- with self.assertRaises((SelectorError, SelectorSyntaxError,)):
- self.page.css('4 ayo')
- with self.assertRaises((SelectorError, SelectorSyntaxError,)):
- self.page.xpath('4 ayo')
-
- def test_selectors_generation(self):
- """Try to create selectors for all elements in the page"""
- def _traverse(element: Adaptor):
- self.assertTrue(type(element.generate_css_selector) is str)
- self.assertTrue(type(element.generate_xpath_selector) is str)
- for branch in element.children:
- _traverse(branch)
-
- _traverse(self.page)
-
- def test_getting_all_text(self):
- """Test getting all text"""
- self.assertNotEqual(self.page.get_all_text(), '')
-
- def test_element_navigation(self):
- """Test moving in the page from selected element"""
- table = self.page.css('.product-list')[0]
-
- self.assertIsNot(table.path, [])
- self.assertNotEqual(table.html_content, '')
- self.assertNotEqual(table.prettify(), '')
+# Navigation and Traversal Tests
+class TestElementNavigation:
+ def test_basic_navigation_properties(self, page):
+ """Test basic navigation properties of elements"""
+ table = page.css('.product-list')[0]
+ assert table.path is not None
+ assert table.html_content != ''
+ assert table.prettify() != ''
+ def test_parent_and_sibling_navigation(self, page):
+ """Test parent and sibling navigation"""
+ table = page.css('.product-list')[0]
parent = table.parent
- self.assertEqual(parent.attrib['id'], 'products')
-
- children = table.children
- self.assertEqual(len(children), 3)
+ assert parent.attrib['id'] == 'products'
parent_siblings = parent.siblings
- self.assertEqual(len(parent_siblings), 1)
+ assert len(parent_siblings) == 1
- child = table.find({'data-id': "1"})
+ def test_child_navigation(self, page):
+ """Test child navigation"""
+ table = page.css('.product-list')[0]
+ children = table.children
+ assert len(children) == 3
+
+ def test_next_and_previous_navigation(self, page):
+ """Test next and previous element navigation"""
+ child = page.css('.product-list')[0].find({'data-id': "1"})
next_element = child.next
- self.assertEqual(next_element.attrib['data-id'], '2')
+ assert next_element.attrib['data-id'] == '2'
prev_element = next_element.previous
- self.assertEqual(prev_element.tag, child.tag)
+ assert prev_element.tag == child.tag
- all_prices = self.page.css('.price')
+ def test_ancestor_finding(self, page):
+ """Test finding ancestors of elements"""
+ all_prices = page.css('.price')
products_with_prices = [
price.find_ancestor(lambda p: p.has_class('product'))
for price in all_prices
]
- self.assertEqual(len(products_with_prices), 3)
+ assert len(products_with_prices) == 3
- def test_empty_return(self):
- """Test cases where functions shouldn't have results"""
- test_html = """
-
-
-
- """
- soup = Adaptor(test_html, auto_match=False, keep_comments=False)
- html_tag = soup.css('html')[0]
- self.assertEqual(html_tag.path, [])
- self.assertEqual(html_tag.siblings, [])
- self.assertEqual(html_tag.parent, None)
- self.assertEqual(html_tag.find_ancestor(lambda e: e), None)
- self.assertEqual(soup.css('#a a')[0].next, None)
- self.assertEqual(soup.css('#b a')[0].previous, None)
-
- def test_text_to_json(self):
- """Test converting text to json"""
- script_content = self.page.css('#page-data::text')[0]
- self.assertTrue(issubclass(type(script_content.sort()), str))
+# JSON and Attribute Tests
+class TestJSONAndAttributes:
+ def test_json_conversion(self, page):
+ """Test converting content to JSON"""
+ script_content = page.css('#page-data::text')[0]
+ assert issubclass(type(script_content.sort()), str)
page_data = script_content.json()
- self.assertEqual(page_data['totalProducts'], 3)
- self.assertTrue('lastUpdated' in page_data)
+ assert page_data['totalProducts'] == 3
+ assert 'lastUpdated' in page_data
- def test_regex_on_text(self):
- """Test doing regex on a selected text"""
- element = self.page.css('[data-id="1"] .price')[0]
- match = element.re_first(r'[\.\d]+')
- self.assertEqual(match, '10.99')
- match = element.text.re(r'(\d+)', replace_entities=False)
- self.assertEqual(len(match), 2)
-
- def test_attribute_operations(self):
- """Test operations on elements attributes"""
- products = self.page.css('.product')
+ def test_attribute_operations(self, page):
+ """Test various attribute-related operations"""
+ # Product ID extraction
+ products = page.css('.product')
product_ids = [product.attrib['data-id'] for product in products]
- self.assertEqual(product_ids, ['1', '2', '3'])
- self.assertTrue('data-id' in products[0].attrib)
+ assert product_ids == ['1', '2', '3']
+ assert 'data-id' in products[0].attrib
- reviews = self.page.css('.review')
+ # Review rating calculations
+ reviews = page.css('.review')
review_ratings = [int(review.attrib['data-rating']) for review in reviews]
- self.assertEqual(sum(review_ratings) / len(review_ratings), 4.5)
+ assert sum(review_ratings) / len(review_ratings) == 4.5
+ # Attribute searching
key_value = list(products[0].attrib.search_values('1', partial=False))
- self.assertEqual(list(key_value[0].keys()), ['data-id'])
+ assert list(key_value[0].keys()) == ['data-id']
key_value = list(products[0].attrib.search_values('1', partial=True))
- self.assertEqual(list(key_value[0].keys()), ['data-id'])
+ assert list(key_value[0].keys()) == ['data-id']
- attr_json = self.page.css_first('#products').attrib['schema'].json()
- self.assertEqual(attr_json, {'jsonable': 'data'})
- self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
-
- def test_performance(self):
- """Test parsing and selecting speed"""
- import time
- large_html = '' + '
+
-
- Great product!
+ John DoeCustomer Reviews
-
-
+
-
- Great product!
- John Doe -
-
+ Good value for money.
- Jane Smith -
+
-
-
-
-
-
-
- '''
- self.page = Adaptor(self.html, auto_match=False)
+ Good value for money.
+ Jane Smith' * 5000 + '
' * 5000 + ''
-
- start_time = time.time()
- parsed = Adaptor(large_html, auto_match=False)
- elements = parsed.css('.item')
- end_time = time.time()
-
- self.assertEqual(len(elements), 5000)
- # Converting 5000 elements to a class and doing operations on them will take time
- # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
- self.assertLess(end_time - start_time, 0.5) # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
+ # JSON attribute conversion
+ attr_json = page.css_first('#products').attrib['schema'].json()
+ assert attr_json == {'jsonable': 'data'}
+ assert isinstance(page.css('#products')[0].attrib.json_string, bytes)
-# Use `coverage run -m unittest --verbose tests/test_parser_functions.py` instead for the coverage report
-# if __name__ == '__main__':
-# unittest.main(verbosity=2)
+# Performance Test
+def test_large_html_parsing_performance():
+ """Test parsing and selecting performance on large HTML"""
+ large_html = '' + '' * 5000 + '
' * 5000 + ''
+
+ start_time = time.time()
+ parsed = Adaptor(large_html, auto_match=False)
+ elements = parsed.css('.item')
+ end_time = time.time()
+
+ assert len(elements) == 5000
+ # Converting 5000 elements to a class and doing operations on them will take time
+ # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
+ assert end_time - start_time < 0.5 # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
+
+
+# Selector Generation Test
+def test_selectors_generation(page):
+ """Try to create selectors for all elements in the page"""
+
+ def _traverse(element: Adaptor):
+ assert isinstance(element.generate_css_selector, str)
+ assert isinstance(element.generate_xpath_selector, str)
+ for branch in element.children:
+ _traverse(branch)
+
+ _traverse(page)
+
+
+# Miscellaneous Tests
+def test_getting_all_text(page):
+ """Test getting all text from the page"""
+ assert page.get_all_text() != ''
+
+
+def test_regex_on_text(page):
+ """Test regex operations on text"""
+ element = page.css('[data-id="1"] .price')[0]
+ match = element.re_first(r'[\.\d]+')
+ assert match == '10.99'
+ match = element.text.re(r'(\d+)', replace_entities=False)
+ assert len(match) == 2