diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.py index 0c5cc86..9594a9c 100644 --- a/tests/fetchers/sync/test_camoufox.py +++ b/tests/fetchers/sync/test_camoufox.py @@ -37,7 +37,7 @@ class TestStealthyFetcher: assert cookies == {"test": "value"} def test_automation(self, fetcher): - """Test if automation break the code or not""" + """Test if automation breaks the code or not""" def scroll_page(page): page.mouse.wheel(10, 0) @@ -59,6 +59,7 @@ class TestStealthyFetcher: { "network_idle": True, "wait": 10, + "timeout": 30_000, "cookies": [], "google_search": True, "extra_headers": {"ayo": ""}, diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 266e72a..6c17a61 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -1,10 +1,12 @@ import pickle import time +import logging import pytest from cssselect import SelectorError, SelectorSyntaxError from scrapling import Selector +logging.getLogger("scrapling").setLevel(logging.DEBUG) @pytest.fixture diff --git a/tests/parser/test_html_utils.py b/tests/parser/test_html_utils.py new file mode 100644 index 0000000..924ef95 --- /dev/null +++ b/tests/parser/test_html_utils.py @@ -0,0 +1,194 @@ +import pytest + +from scrapling.core._html_utils import to_unicode, _replace_entities, name2codepoint + + +class TestToUnicode: + def test_string_input(self): + """Test to_unicode with string input""" + text = "hello world" + assert to_unicode(text) == "hello world" + + def test_bytes_input_default_encoding(self): + """Test to_unicode with `bytes` input using default UTF-8""" + text = b"hello world" + assert to_unicode(text) == "hello world" + + def test_bytes_input_custom_encoding(self): + """Test to_unicode with custom encoding""" + text = "café".encode('latin-1') + assert to_unicode(text, encoding='latin-1') == "café" + + def test_bytes_input_with_errors(self): + """Test to_unicode with error handling""" + # Invalid UTF-8 bytes + text = b'\xff\xfe' + assert to_unicode(text, errors='ignore') == "" + assert to_unicode(text, errors='replace') == "��" + + def test_invalid_input_type(self): + """Test to_unicode with an invalid input type""" + with pytest.raises(TypeError, match="to_unicode must receive bytes or str"): + to_unicode(123) + + def test_none_encoding_defaults_to_utf8(self): + """Test that None encoding defaults to UTF-8""" + text = "café".encode('utf-8') + assert to_unicode(text, encoding=None) == "café" + + +class TestReplaceEntities: + def test_named_entities(self): + """Test replacement of named HTML entities""" + text = "& < > "  " + result = _replace_entities(text) + assert result == "& < > \" \xa0" + + def test_decimal_entities(self): + """Test replacement of decimal numeric entities""" + text = "& < >" + result = _replace_entities(text) + assert result == "& < >" + + def test_hexadecimal_entities(self): + """Test replacement of hexadecimal numeric entities""" + text = "& < >" + result = _replace_entities(text) + assert result == "& < >" + + def test_mixed_entities(self): + """Test replacement of mixed entity types""" + text = "Price: £100 €50 $25" + result = _replace_entities(text) + assert result == "Price: £100 €50 $25" + + def test_keep_entities(self): + """Test keeping specific entities""" + text = "& < >" + result = _replace_entities(text, keep=['amp', 'lt']) + assert result == "& < >" + + def test_windows_1252_range(self): + """Test handling of Windows-1252 range characters""" + text = "€ ‚ Ÿ" # Windows-1252 range + result = _replace_entities(text) + # These should be decoded using cp1252 + assert "€" in result # 128 -> Euro sign + + def test_remove_illegal_entities_true(self): + """Test removing illegal entities with remove_illegal=True""" + text = "&unknown; 󴈿" + result = _replace_entities(text, remove_illegal=True) + # The function may convert large numbers to Unicode characters or leave them as-is + assert "&unknown;" not in result # Unknown entities should be removed or converted + + def test_remove_illegal_entities_false(self): + """Test keeping illegal entities with remove_illegal=False""" + text = "&unknown; 󴈿" + result = _replace_entities(text, remove_illegal=False) + # Unknown entities should be preserved when remove_illegal=False + assert "&unknown;" in result + # Large numeric entities may be converted to Unicode characters + + def test_bytes_input(self): + """Test with bytes input""" + text = b"& < >" + result = _replace_entities(text) + assert result == "& < >" + + def test_custom_encoding(self): + """Test with custom encoding""" + text = "é".encode('latin-1') + result = _replace_entities(text, encoding='latin-1') + assert result == "é" + + def test_entities_without_semicolon(self): + """Test entities without semicolon""" + text = "& < >" + result = _replace_entities(text, remove_illegal=True) + # Should handle entities without a semicolon + assert len(result) <= len(text) + + def test_case_insensitive_named_entities(self): + """Test case-insensitive named-entity handling""" + text = "& ≪ >" + result = _replace_entities(text) + assert result == "& < >" + + def test_edge_cases(self): + """Test edge cases""" + # Empty string + assert _replace_entities("") == "" + + # No entities + assert _replace_entities("plain text") == "plain text" + + # Invalid numeric entity + text = "&#-1;" + result = _replace_entities(text, remove_illegal=True) + # Invalid entities may be left as-is or removed depending on implementation + assert len(result) >= 0 # Ensure no exception is raised + + +class TestName2Codepoint: + def test_common_entities_exist(self): + """Test that common HTML entities exist in mapping""" + common_entities = ['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg'] + for entity in common_entities: + assert entity in name2codepoint + + def test_greek_letters_exist(self): + """Test that Greek letter entities exist""" + greek_letters = ['alpha', 'beta', 'gamma', 'delta', 'epsilon'] + for letter in greek_letters: + assert letter in name2codepoint + + def test_mathematical_symbols_exist(self): + """Test that mathematical symbol entities exist""" + math_symbols = ['sum', 'prod', 'int', 'infin', 'plusmn'] + for symbol in math_symbols: + assert symbol in name2codepoint + + def test_currency_symbols_exist(self): + """Test that currency symbol entities exist""" + currencies = ['pound', 'yen', 'euro', 'cent'] + for currency in currencies: + assert currency in name2codepoint + + def test_codepoint_values(self): + """Test specific codepoint values""" + assert name2codepoint['amp'] == 0x0026 # & + assert name2codepoint['lt'] == 0x003C # < + assert name2codepoint['gt'] == 0x003E # > + assert name2codepoint['nbsp'] == 0x00A0 # non-breaking space + assert name2codepoint['copy'] == 0x00A9 # © + + +class TestIntegration: + def test_real_world_html(self): + """Test with real-world HTML content""" + html = """ + <div class="content"> + © 2024 Company & Associates + Price: £99.99 (€89.99) + Math: α + β = γ + </div> + """ + result = _replace_entities(html) + + assert '
' in result + assert '© 2024 Company & Associates' in result + assert 'Price: £99.99 (€89.99)' in result + assert 'Math: α + β = γ' in result + + def test_performance_with_large_text(self): + """Test performance with large text containing many entities""" + # Create large text with repeated entities + text = ("& < > " " * 1000) + result = _replace_entities(text) + + # Should complete without issues and have correct content + assert result.count("&") == 1000 + assert result.count("<") == 1000 + assert result.count(">") == 1000 + assert result.count('"') == 1000