diff --git a/tests/ai/__init__.py b/tests/ai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py new file mode 100644 index 0000000..92f5886 --- /dev/null +++ b/tests/ai/test_ai_mcp.py @@ -0,0 +1,89 @@ +import pytest +from unittest.mock import Mock, patch + +from scrapling.core.ai import ScraplingMCPServer, ResponseModel + + +class TestMCPServer: + """Test MCP server functionality""" + + @pytest.fixture + def server(self): + return ScraplingMCPServer() + + def test_server_creation(self, server): + """Test server instance creation""" + assert server._server is not None + assert server._server.name == "Scrapling" + + def test_get_tool(self): + """Test the get tool method""" + with patch('scrapling.fetchers.Fetcher.get') as mock_get: + mock_response = Mock() + mock_response.status = 200 + mock_response.url = "https://example.com" + mock_get.return_value = mock_response + + with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract: + mock_extract.return_value = iter(["Content"]) + + result = ScraplingMCPServer.get( + url="https://example.com", + extraction_type="markdown" + ) + + assert isinstance(result, ResponseModel) + assert result.status == 200 + assert result.url == "https://example.com" + + @pytest.mark.asyncio + async def test_bulk_get_tool(self): + """Test the bulk_get tool method""" + with patch('scrapling.engines.FetcherSession') as mock_session: + mock_instance = Mock() + mock_session.return_value.__aenter__.return_value = mock_instance + + # Mock async get method + async def mock_async_get(*args, **kwargs): + mock_resp = Mock() + mock_resp.status = 200 + mock_resp.url = args[0] + return mock_resp + + mock_instance.get = mock_async_get + + with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract: + mock_extract.return_value = iter(["Content"]) + + results = await ScraplingMCPServer.bulk_get( + urls=("https://example1.com", "https://example2.com"), + extraction_type="html" + ) + + assert len(results) == 2 + assert all(isinstance(r, ResponseModel) for r in results) + + @pytest.mark.asyncio + async def test_fetch_tool(self): + """Test the fetch tool method""" + with patch('scrapling.fetchers.DynamicFetcher.async_fetch') as mock_fetch: + mock_response = Mock() + mock_response.status = 200 + mock_response.url = "https://example.com" + mock_fetch.return_value = mock_response + + with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract: + mock_extract.return_value = iter(["Content"]) + + result = await ScraplingMCPServer.fetch( + url="https://example.com", + headless=True + ) + + assert isinstance(result, ResponseModel) + + def test_serve_method(self, server): + """Test the serve method""" + with patch.object(server._server, 'run') as mock_run: + server.serve() + mock_run.assert_called_once_with(transport="stdio") diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py new file mode 100644 index 0000000..7ff0f18 --- /dev/null +++ b/tests/fetchers/test_response_handling.py @@ -0,0 +1,109 @@ +from unittest.mock import Mock + +from scrapling.parser import Selector +from scrapling.engines.toolbelt import ResponseFactory, Response +from scrapling.engines.toolbelt.custom import ResponseEncoding + + +class TestResponseFactory: + """Test ResponseFactory functionality""" + + def test_response_from_curl(self): + """Test creating response from curl_cffi response""" + # Mock curl response + mock_curl_response = Mock() + mock_curl_response.url = "https://example.com" + mock_curl_response.content = b"Test" + mock_curl_response.status_code = 200 + mock_curl_response.reason = "OK" + mock_curl_response.encoding = "utf-8" + mock_curl_response.cookies = {"session": "abc"} + mock_curl_response.headers = {"Content-Type": "text/html"} + mock_curl_response.request.headers = {"User-Agent": "Test"} + mock_curl_response.request.method = "GET" + mock_curl_response.history = [] + + response = ResponseFactory.from_http_request( + mock_curl_response, + {"adaptive": False} + ) + + assert response.status == 200 + assert response.url == "https://example.com" + assert isinstance(response, Response) + + def test_response_encoding_edge_cases(self): + """Test response encoding handling""" + # Test various content types + test_cases = [ + (None, "utf-8"), + ("", "utf-8"), + ("text/html; charset=invalid", "utf-8"), + ("application/octet-stream", "utf-8"), + ] + + for content_type, expected in test_cases: + encoding = ResponseEncoding.get_value(content_type) + assert encoding == expected + + def test_response_history_processing(self): + """Test processing response history""" + # Mock responses with redirects + mock_final = Mock() + mock_final.status = 200 + mock_final.status_text = "OK" + mock_final.all_headers = Mock(return_value={}) + + mock_redirect = Mock() + mock_redirect.url = "https://example.com/redirect" + mock_redirect.response = Mock(return_value=mock_final) + mock_redirect.all_headers = Mock(return_value={}) + mock_redirect.redirected_from = None + + mock_first = Mock() + mock_first.request.redirected_from = mock_redirect + + # Process history + history = ResponseFactory._process_response_history( + mock_first, + {} + ) + + assert len(history) >= 0 # Should process redirects + + +class TestErrorScenarios: + """Test various error scenarios""" + + def test_invalid_html_handling(self): + """Test handling of malformed HTML""" + malformed_html = """ + + +
Unclosed div +

Paragraph without closing tag + Nested unclosed + + """ + + # Should handle gracefully + page = Selector(malformed_html) + assert page is not None + + # Should still be able to select elements + divs = page.css("div") + assert len(divs) > 0 + + def test_empty_responses(self): + """Test handling of empty responses""" + # Empty HTML + page = Selector("") + assert page is not None + + # Whitespace only + page = Selector(" \n\t ") + assert page is not None + + # Null bytes + page = Selector("Hello\x00World") + assert "Hello" in page.get_all_text() diff --git a/tests/parser/test_attributes_handler.py b/tests/parser/test_attributes_handler.py new file mode 100644 index 0000000..6827a95 --- /dev/null +++ b/tests/parser/test_attributes_handler.py @@ -0,0 +1,336 @@ +import pytest +import json + +from scrapling import Selector +from scrapling.core.custom_types import AttributesHandler + + +class TestAttributesHandler: + """Test AttributesHandler functionality""" + + @pytest.fixture + def sample_html(self): + return """ + + +

+ Content +
+ + Photo + + + """ + + @pytest.fixture + def attributes(self, sample_html): + page = Selector(sample_html) + element = page.css("#main")[0] + return element.attrib + + def test_basic_attribute_access(self, attributes): + """Test basic attribute access""" + # Dict-like access + assert attributes["id"] == "main" + assert attributes["class"] == "container active" + assert attributes["title"] == "Main Container" + + # Key existence + assert "id" in attributes + assert "nonexistent" not in attributes + + # Get with default + assert attributes.get("id") == "main" + assert attributes.get("nonexistent") is None + assert attributes.get("nonexistent", "default") == "default" + + def test_iteration_methods(self, attributes): + """Test iteration over attributes""" + # Keys + keys = list(attributes.keys()) + assert "id" in keys + assert "class" in keys + assert "data-config" in keys + + # Values + values = list(attributes.values()) + assert "main" in values + assert "container active" in values + + # Items + items = dict(attributes.items()) + assert items["id"] == "main" + assert items["class"] == "container active" + + # Length + assert len(attributes) > 0 + + def test_json_parsing(self, attributes): + """Test JSON parsing from attributes""" + # Valid JSON object + config = attributes["data-config"].json() + assert config["theme"] == "dark" + assert config["version"] == 2.5 + + # Valid JSON array + items = attributes["data-items"].json() + assert items == [1, 2, 3, 4, 5] + + # Nested JSON + nested = attributes["data-nested"].json() + assert nested["user"]["name"] == "John" + assert nested["user"]["age"] == 30 + + # JSON null + assert attributes["data-null"].json() is None + + def test_json_error_handling(self, attributes): + """Test JSON parsing error handling""" + # Invalid JSON should raise error or return None + with pytest.raises((json.JSONDecodeError, AttributeError)): + attributes["data-invalid-json"].json() + + # Non-existent attribute + with pytest.raises(KeyError): + attributes["nonexistent"].json() + + def test_json_string_property(self, attributes): + """Test json_string property""" + # Should return JSON representation of all attributes + json_string = attributes.json_string + assert isinstance(json_string, bytes) + + # Parse it back + parsed = json.loads(json_string) + assert parsed["id"] == "main" + assert parsed["class"] == "container active" + + def test_search_values(self, attributes): + """Test search_values method""" + # Exact match + results = list(attributes.search_values("main", partial=False)) + assert len(results) == 1 + assert "id" in results[0] + + # Partial match + results = list(attributes.search_values("container", partial=True)) + assert len(results) >= 1 + found_keys = [] + for result in results: + found_keys.extend(result.keys()) + assert "class" in found_keys or "title" in found_keys + + # Case sensitivity + results = list(attributes.search_values("MAIN", partial=False)) + assert len(results) == 0 # Should be case-sensitive by default + + # Multiple matches + results = list(attributes.search_values("2", partial=True)) + assert len(results) > 1 # Should find multiple attributes + + # No matches + results = list(attributes.search_values("nonexistent", partial=False)) + assert len(results) == 0 + + def test_special_attribute_types(self, sample_html): + """Test handling of special attribute types""" + page = Selector(sample_html) + + # Boolean attributes + input_elem = page.css("input")[0] + assert "required" in input_elem.attrib + assert "disabled" in input_elem.attrib + + # Empty attributes + main_elem = page.css("#main")[0] + assert main_elem.attrib["data-empty"] == "" + + # Numeric string attributes + assert main_elem.attrib["data-number"] == "42" + assert main_elem.attrib["data-bool"] == "true" + + def test_attribute_modification(self, sample_html): + """Test that AttributesHandler is read-only (if applicable)""" + page = Selector(sample_html) + element = page.css("#main")[0] + attrs = element.attrib + + # Test if attributes can be modified + # This behavior depends on implementation + original_id = attrs["id"] + try: + attrs["id"] = "new-id" + # If modification is allowed + assert attrs["id"] == "new-id" + # Reset + attrs["id"] = original_id + except (TypeError, AttributeError): + # If modification is not allowed (read-only) + assert attrs["id"] == original_id + + def test_string_representation(self, attributes): + """Test string representations""" + # __str__ + str_repr = str(attributes) + assert isinstance(str_repr, str) + assert "id" in str_repr or "main" in str_repr + + # __repr__ + repr_str = repr(attributes) + assert isinstance(repr_str, str) + + def test_edge_cases(self, sample_html): + """Test edge cases and special scenarios""" + page = Selector(sample_html) + + # Element with no attributes + page_with_no_attrs = Selector("
Content
") + elem = page_with_no_attrs.css("div")[0] + assert len(elem.attrib) == 0 + assert list(elem.attrib.keys()) == [] + assert elem.attrib.get("any") is None + + # Element with encoded content + main_elem = page.css("#main")[0] + encoded = main_elem.attrib["data-encoded"] + assert "<" in encoded # Should decode it + + # Style attribute parsing + style = main_elem.attrib["style"] + assert "color: red" in style + assert "background: blue" in style + + def test_url_attribute(self, attributes): + """Test URL attributes""" + url = attributes["data-url"] + assert url == "https://example.com/page?param=value" + + # Could test URL joining if AttributesHandler supports it + # based on the parent element's base URL + + def test_comparison_operations(self, sample_html): + """Test comparison operations if supported""" + page = Selector(sample_html) + elem1 = page.css("#main")[0] + elem2 = page.css("input")[0] + + # Different elements should have different attributes + assert elem1.attrib != elem2.attrib + + # The same element should have equal attributes + elem1_again = page.css("#main")[0] + assert elem1.attrib == elem1_again.attrib + + def test_complex_search_patterns(self, attributes): + """Test complex search patterns""" + # Search for JSON-containing attributes + json_attrs = [] + for key, value in attributes.items(): + try: + if isinstance(value, str) and (value.startswith('{') or value.startswith('[')): + json.loads(value) + json_attrs.append(key) + except: + pass + + assert "data-config" in json_attrs + assert "data-items" in json_attrs + assert "data-nested" in json_attrs + + def test_attribute_filtering(self, attributes): + """Test filtering attributes by patterns""" + # Get all data-* attributes + data_attrs = {k: v for k, v in attributes.items() if k.startswith("data-")} + assert len(data_attrs) > 5 + assert "data-config" in data_attrs + assert "data-items" in data_attrs + + # Get all event handler attributes + event_attrs = {k: v for k, v in attributes.items() if k.startswith("on")} + assert "onclick" in event_attrs + + def test_performance_with_many_attributes(self): + """Test performance with elements having many attributes""" + # Create an element with many attributes + attrs_list = [f'data-attr{i}="value{i}"' for i in range(100)] + html = f'
Content
' + + page = Selector(html) + element = page.css("#test")[0] + attribs = element.attrib + + # Should handle many attributes efficiently + assert len(attribs) == 101 # id + 100 data attributes + + # Search should still work efficiently + results = list(attribs.search_values("value50", partial=False)) + assert len(results) == 1 + + def test_unicode_attributes(self): + """Test handling of Unicode in attributes""" + html = """ +
+
+ """ + + page = Selector(html) + attrs = page.css("#unicode-test")[0].attrib + + assert attrs["data-emoji"] == "😀🎉" + assert attrs["data-chinese"] == "你好世界" + assert attrs["data-arabic"] == "مرحبا بالعالم" + assert attrs["data-special"] == "café naïve" + + # Search with Unicode + results = list(attrs.search_values("你好", partial=True)) + assert len(results) == 1 + + def test_malformed_attributes(self): + """Test handling of malformed attributes""" + # Various malformed HTML scenarios + test_cases = [ + '
Content
', # Empty attribute value + '
Content
', # No attribute value + '
Content
', # Invalid attribute name + '
Content
', # Unquoted values + ] + + for html in test_cases: + try: + page = Selector(html) + if page.css("div"): + attrs = page.css("div")[0].attrib + # Should handle gracefully without crashing + assert isinstance(attrs, AttributesHandler) + except: + # Some malformed HTML might not parse at all + pass diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py new file mode 100644 index 0000000..aa9ea7a --- /dev/null +++ b/tests/parser/test_parser_advanced.py @@ -0,0 +1,224 @@ +import re +import pytest + +from scrapling import Selector, Selectors +from scrapling.core.custom_types import TextHandler, TextHandlers + + +class TestAdvancedSelectors: + """Test advanced selector functionality""" + + @pytest.fixture + def complex_html(self): + return """ + + +
+

First paragraph

+ +

Second paragraph

+ +
+ Special content + Regular content +
+ + + +
Cell 1Cell 2
Cell 3Cell 4
+
+ + + """ + + def test_comment_and_cdata_handling(self, complex_html): + """Test handling of comments and CDATA""" + # With comments/CDATA kept + page = Selector( + complex_html, + keep_comments=True, + keep_cdata=True + ) + content = page.body + assert "Comment" in content + assert "CDATA" in content + + # Without comments/CDATA + page = Selector( + complex_html, + keep_comments=False, + keep_cdata=False + ) + content = page.body + assert "Comment" not in content + + def test_advanced_xpath_variables(self, complex_html): + """Test XPath with variables""" + page = Selector(complex_html) + + # Using XPath variables + cells = page.xpath( + "//td[text()=$cell_text]", + cell_text="Cell 1" + ) + assert len(cells) == 1 + assert cells[0].text == "Cell 1" + + def test_pseudo_elements(self, complex_html): + """Test CSS pseudo-elements""" + page = Selector(complex_html) + + # ::text pseudo-element + texts = page.css("p::text") + assert len(texts) == 2 + assert isinstance(texts[0], TextHandler) + + # ::attr() pseudo-element + attrs = page.css("div::attr(class)") + assert "container" in attrs + + def test_complex_attribute_operations(self, complex_html): + """Test complex attribute handling""" + page = Selector(complex_html) + container = page.css(".container")[0] + + # JSON in attributes + data = container.attrib["data-test"].json() + assert data["key"] == "value" + + # Attribute searching + matches = list(container.attrib.search_values("container")) + assert len(matches) == 1 + + def test_url_joining(self): + """Test URL joining functionality""" + page = Selector("", url="https://example.com/page") + + # Relative URL + assert page.urljoin("../other") == "https://example.com/other" + assert page.urljoin("/absolute") == "https://example.com/absolute" + assert page.urljoin("relative") == "https://example.com/relative" + + def test_find_operations_edge_cases(self, complex_html): + """Test edge cases in find operations""" + page = Selector(complex_html) + + # Multiple argument types + _ = page.find_all( + "span", + ["div"], + {"class": "nested"}, + lambda e: e.text != "" + ) + + # Regex pattern matching + pattern = re.compile(r"Cell \d+") + cells = page.find_all(pattern) + assert len(cells) == 4 + + def test_text_operations_edge_cases(self, complex_html): + """Test text operation edge cases""" + page = Selector(complex_html) + + # get_all_text with a custom separator + text = page.get_all_text(separator=" | ", strip=True) + assert " | " in text + + # Ignore specific tags + text = page.get_all_text(ignore_tags=("table",)) + assert "Cell" not in text + + # With empty values + text = page.get_all_text(valid_values=False) + assert text != "" + + +class TestTextHandlerAdvanced: + """Test advanced TextHandler functionality""" + + def test_text_handler_operations(self): + """Test various TextHandler operations""" + text = TextHandler(" Hello World ") + + # All string methods should return TextHandler + assert isinstance(text.strip(), TextHandler) + assert isinstance(text.upper(), TextHandler) + assert isinstance(text.lower(), TextHandler) + assert isinstance(text.replace("World", "Python"), TextHandler) + + # Custom methods + assert text.clean() == "Hello World" + + # Sorting + text2 = TextHandler("dcba") + assert text2.sort() == "abcd" + + def test_text_handler_regex(self): + """Test regex operations on TextHandler""" + text = TextHandler("Price: $10.99, Sale: $8.99") + + # Basic regex + prices = text.re(r"\$[\d.]+") + assert len(prices) == 2 + assert prices[0] == "$10.99" + + # Case insensitive + text2 = TextHandler("HELLO hello HeLLo") + matches = text2.re(r"hello", case_sensitive=False) + assert len(matches) == 3 + + # Clean match + text3 = TextHandler(" He l lo ") + matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False) + assert len(matches) == 1 + + def test_text_handlers_operations(self): + """Test TextHandlers list operations""" + handlers = TextHandlers([ + TextHandler("First"), + TextHandler("Second"), + TextHandler("Third") + ]) + + # Slicing should return TextHandlers + assert isinstance(handlers[0:2], TextHandlers) + + # Get methods + assert handlers.get() == "First" + assert handlers.get("default") == "First" + assert TextHandlers([]).get("default") == "default" + + +class TestSelectorsAdvanced: + """Test advanced Selectors functionality""" + + def test_selectors_filtering(self): + """Test filtering operations on Selectors""" + html = """ +
+

Important

+

Regular

+

Also important

+
+ """ + page = Selector(html) + paragraphs = page.css("p") + + # Filter by class + highlighted = paragraphs.filter(lambda p: p.has_class("highlight")) + assert len(highlighted) == 2 + + # Search for a specific element + found = paragraphs.search(lambda p: p.text == "Regular") + assert found is not None + assert found.text == "Regular" + + def test_selectors_properties(self): + """Test Selectors properties""" + html = "

1

2

3

" + page = Selector(html) + paragraphs = page.css("p") + + assert paragraphs.first.text == "1" + assert paragraphs.last.text == "3" + assert paragraphs.length == 3