diff --git a/tests/parser/test_html_utils.py b/tests/parser/test_html_utils.py deleted file mode 100644 index 924ef95..0000000 --- a/tests/parser/test_html_utils.py +++ /dev/null @@ -1,194 +0,0 @@ -import pytest - -from scrapling.core._html_utils import to_unicode, _replace_entities, name2codepoint - - -class TestToUnicode: - def test_string_input(self): - """Test to_unicode with string input""" - text = "hello world" - assert to_unicode(text) == "hello world" - - def test_bytes_input_default_encoding(self): - """Test to_unicode with `bytes` input using default UTF-8""" - text = b"hello world" - assert to_unicode(text) == "hello world" - - def test_bytes_input_custom_encoding(self): - """Test to_unicode with custom encoding""" - text = "café".encode('latin-1') - assert to_unicode(text, encoding='latin-1') == "café" - - def test_bytes_input_with_errors(self): - """Test to_unicode with error handling""" - # Invalid UTF-8 bytes - text = b'\xff\xfe' - assert to_unicode(text, errors='ignore') == "" - assert to_unicode(text, errors='replace') == "��" - - def test_invalid_input_type(self): - """Test to_unicode with an invalid input type""" - with pytest.raises(TypeError, match="to_unicode must receive bytes or str"): - to_unicode(123) - - def test_none_encoding_defaults_to_utf8(self): - """Test that None encoding defaults to UTF-8""" - text = "café".encode('utf-8') - assert to_unicode(text, encoding=None) == "café" - - -class TestReplaceEntities: - def test_named_entities(self): - """Test replacement of named HTML entities""" - text = "& < > "  " - result = _replace_entities(text) - assert result == "& < > \" \xa0" - - def test_decimal_entities(self): - """Test replacement of decimal numeric entities""" - text = "& < >" - result = _replace_entities(text) - assert result == "& < >" - - def test_hexadecimal_entities(self): - """Test replacement of hexadecimal numeric entities""" - text = "& < >" - result = _replace_entities(text) - assert result == "& < >" - - def test_mixed_entities(self): - """Test replacement of mixed entity types""" - text = "Price: £100 €50 $25" - result = _replace_entities(text) - assert result == "Price: £100 €50 $25" - - def test_keep_entities(self): - """Test keeping specific entities""" - text = "& < >" - result = _replace_entities(text, keep=['amp', 'lt']) - assert result == "& < >" - - def test_windows_1252_range(self): - """Test handling of Windows-1252 range characters""" - text = "€ ‚ Ÿ" # Windows-1252 range - result = _replace_entities(text) - # These should be decoded using cp1252 - assert "€" in result # 128 -> Euro sign - - def test_remove_illegal_entities_true(self): - """Test removing illegal entities with remove_illegal=True""" - text = "&unknown; 󴈿" - result = _replace_entities(text, remove_illegal=True) - # The function may convert large numbers to Unicode characters or leave them as-is - assert "&unknown;" not in result # Unknown entities should be removed or converted - - def test_remove_illegal_entities_false(self): - """Test keeping illegal entities with remove_illegal=False""" - text = "&unknown; 󴈿" - result = _replace_entities(text, remove_illegal=False) - # Unknown entities should be preserved when remove_illegal=False - assert "&unknown;" in result - # Large numeric entities may be converted to Unicode characters - - def test_bytes_input(self): - """Test with bytes input""" - text = b"& < >" - result = _replace_entities(text) - assert result == "& < >" - - def test_custom_encoding(self): - """Test with custom encoding""" - text = "é".encode('latin-1') - result = _replace_entities(text, encoding='latin-1') - assert result == "é" - - def test_entities_without_semicolon(self): - """Test entities without semicolon""" - text = "& < >" - result = _replace_entities(text, remove_illegal=True) - # Should handle entities without a semicolon - assert len(result) <= len(text) - - def test_case_insensitive_named_entities(self): - """Test case-insensitive named-entity handling""" - text = "& ≪ >" - result = _replace_entities(text) - assert result == "& < >" - - def test_edge_cases(self): - """Test edge cases""" - # Empty string - assert _replace_entities("") == "" - - # No entities - assert _replace_entities("plain text") == "plain text" - - # Invalid numeric entity - text = "&#-1;" - result = _replace_entities(text, remove_illegal=True) - # Invalid entities may be left as-is or removed depending on implementation - assert len(result) >= 0 # Ensure no exception is raised - - -class TestName2Codepoint: - def test_common_entities_exist(self): - """Test that common HTML entities exist in mapping""" - common_entities = ['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg'] - for entity in common_entities: - assert entity in name2codepoint - - def test_greek_letters_exist(self): - """Test that Greek letter entities exist""" - greek_letters = ['alpha', 'beta', 'gamma', 'delta', 'epsilon'] - for letter in greek_letters: - assert letter in name2codepoint - - def test_mathematical_symbols_exist(self): - """Test that mathematical symbol entities exist""" - math_symbols = ['sum', 'prod', 'int', 'infin', 'plusmn'] - for symbol in math_symbols: - assert symbol in name2codepoint - - def test_currency_symbols_exist(self): - """Test that currency symbol entities exist""" - currencies = ['pound', 'yen', 'euro', 'cent'] - for currency in currencies: - assert currency in name2codepoint - - def test_codepoint_values(self): - """Test specific codepoint values""" - assert name2codepoint['amp'] == 0x0026 # & - assert name2codepoint['lt'] == 0x003C # < - assert name2codepoint['gt'] == 0x003E # > - assert name2codepoint['nbsp'] == 0x00A0 # non-breaking space - assert name2codepoint['copy'] == 0x00A9 # © - - -class TestIntegration: - def test_real_world_html(self): - """Test with real-world HTML content""" - html = """ - <div class="content"> - © 2024 Company & Associates - Price: £99.99 (€89.99) - Math: α + β = γ - </div> - """ - result = _replace_entities(html) - - assert '
' in result - assert '© 2024 Company & Associates' in result - assert 'Price: £99.99 (€89.99)' in result - assert 'Math: α + β = γ' in result - - def test_performance_with_large_text(self): - """Test performance with large text containing many entities""" - # Create large text with repeated entities - text = ("& < > " " * 1000) - result = _replace_entities(text) - - # Should complete without issues and have correct content - assert result.count("&") == 1000 - assert result.count("<") == 1000 - assert result.count(">") == 1000 - assert result.count('"') == 1000