test: adding new tests and updating existing ones
The coverage is now 78%
This commit is contained in:
Karim shoair
@@ -0,0 +1,89 @@
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from scrapling.core.ai import ScraplingMCPServer, ResponseModel
|
||||
|
||||
|
||||
class TestMCPServer:
|
||||
"""Test MCP server functionality"""
|
||||
|
||||
@pytest.fixture
|
||||
def server(self):
|
||||
return ScraplingMCPServer()
|
||||
|
||||
def test_server_creation(self, server):
|
||||
"""Test server instance creation"""
|
||||
assert server._server is not None
|
||||
assert server._server.name == "Scrapling"
|
||||
|
||||
def test_get_tool(self):
|
||||
"""Test the get tool method"""
|
||||
with patch('scrapling.fetchers.Fetcher.get') as mock_get:
|
||||
mock_response = Mock()
|
||||
mock_response.status = 200
|
||||
mock_response.url = "https://example.com"
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
|
||||
mock_extract.return_value = iter(["Content"])
|
||||
|
||||
result = ScraplingMCPServer.get(
|
||||
url="https://example.com",
|
||||
extraction_type="markdown"
|
||||
)
|
||||
|
||||
assert isinstance(result, ResponseModel)
|
||||
assert result.status == 200
|
||||
assert result.url == "https://example.com"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_get_tool(self):
|
||||
"""Test the bulk_get tool method"""
|
||||
with patch('scrapling.engines.FetcherSession') as mock_session:
|
||||
mock_instance = Mock()
|
||||
mock_session.return_value.__aenter__.return_value = mock_instance
|
||||
|
||||
# Mock async get method
|
||||
async def mock_async_get(*args, **kwargs):
|
||||
mock_resp = Mock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.url = args[0]
|
||||
return mock_resp
|
||||
|
||||
mock_instance.get = mock_async_get
|
||||
|
||||
with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
|
||||
mock_extract.return_value = iter(["Content"])
|
||||
|
||||
results = await ScraplingMCPServer.bulk_get(
|
||||
urls=("https://example1.com", "https://example2.com"),
|
||||
extraction_type="html"
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
assert all(isinstance(r, ResponseModel) for r in results)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_tool(self):
|
||||
"""Test the fetch tool method"""
|
||||
with patch('scrapling.fetchers.DynamicFetcher.async_fetch') as mock_fetch:
|
||||
mock_response = Mock()
|
||||
mock_response.status = 200
|
||||
mock_response.url = "https://example.com"
|
||||
mock_fetch.return_value = mock_response
|
||||
|
||||
with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
|
||||
mock_extract.return_value = iter(["Content"])
|
||||
|
||||
result = await ScraplingMCPServer.fetch(
|
||||
url="https://example.com",
|
||||
headless=True
|
||||
)
|
||||
|
||||
assert isinstance(result, ResponseModel)
|
||||
|
||||
def test_serve_method(self, server):
|
||||
"""Test the serve method"""
|
||||
with patch.object(server._server, 'run') as mock_run:
|
||||
server.serve()
|
||||
mock_run.assert_called_once_with(transport="stdio")
|
||||
@@ -0,0 +1,109 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
from scrapling.parser import Selector
|
||||
from scrapling.engines.toolbelt import ResponseFactory, Response
|
||||
from scrapling.engines.toolbelt.custom import ResponseEncoding
|
||||
|
||||
|
||||
class TestResponseFactory:
|
||||
"""Test ResponseFactory functionality"""
|
||||
|
||||
def test_response_from_curl(self):
|
||||
"""Test creating response from curl_cffi response"""
|
||||
# Mock curl response
|
||||
mock_curl_response = Mock()
|
||||
mock_curl_response.url = "https://example.com"
|
||||
mock_curl_response.content = b"<html><body>Test</body></html>"
|
||||
mock_curl_response.status_code = 200
|
||||
mock_curl_response.reason = "OK"
|
||||
mock_curl_response.encoding = "utf-8"
|
||||
mock_curl_response.cookies = {"session": "abc"}
|
||||
mock_curl_response.headers = {"Content-Type": "text/html"}
|
||||
mock_curl_response.request.headers = {"User-Agent": "Test"}
|
||||
mock_curl_response.request.method = "GET"
|
||||
mock_curl_response.history = []
|
||||
|
||||
response = ResponseFactory.from_http_request(
|
||||
mock_curl_response,
|
||||
{"adaptive": False}
|
||||
)
|
||||
|
||||
assert response.status == 200
|
||||
assert response.url == "https://example.com"
|
||||
assert isinstance(response, Response)
|
||||
|
||||
def test_response_encoding_edge_cases(self):
|
||||
"""Test response encoding handling"""
|
||||
# Test various content types
|
||||
test_cases = [
|
||||
(None, "utf-8"),
|
||||
("", "utf-8"),
|
||||
("text/html; charset=invalid", "utf-8"),
|
||||
("application/octet-stream", "utf-8"),
|
||||
]
|
||||
|
||||
for content_type, expected in test_cases:
|
||||
encoding = ResponseEncoding.get_value(content_type)
|
||||
assert encoding == expected
|
||||
|
||||
def test_response_history_processing(self):
|
||||
"""Test processing response history"""
|
||||
# Mock responses with redirects
|
||||
mock_final = Mock()
|
||||
mock_final.status = 200
|
||||
mock_final.status_text = "OK"
|
||||
mock_final.all_headers = Mock(return_value={})
|
||||
|
||||
mock_redirect = Mock()
|
||||
mock_redirect.url = "https://example.com/redirect"
|
||||
mock_redirect.response = Mock(return_value=mock_final)
|
||||
mock_redirect.all_headers = Mock(return_value={})
|
||||
mock_redirect.redirected_from = None
|
||||
|
||||
mock_first = Mock()
|
||||
mock_first.request.redirected_from = mock_redirect
|
||||
|
||||
# Process history
|
||||
history = ResponseFactory._process_response_history(
|
||||
mock_first,
|
||||
{}
|
||||
)
|
||||
|
||||
assert len(history) >= 0 # Should process redirects
|
||||
|
||||
|
||||
class TestErrorScenarios:
|
||||
"""Test various error scenarios"""
|
||||
|
||||
def test_invalid_html_handling(self):
|
||||
"""Test handling of malformed HTML"""
|
||||
malformed_html = """
|
||||
<html>
|
||||
<body>
|
||||
<div>Unclosed div
|
||||
<p>Paragraph without closing tag
|
||||
<span>Nested unclosed
|
||||
</body>
|
||||
"""
|
||||
|
||||
# Should handle gracefully
|
||||
page = Selector(malformed_html)
|
||||
assert page is not None
|
||||
|
||||
# Should still be able to select elements
|
||||
divs = page.css("div")
|
||||
assert len(divs) > 0
|
||||
|
||||
def test_empty_responses(self):
|
||||
"""Test handling of empty responses"""
|
||||
# Empty HTML
|
||||
page = Selector("")
|
||||
assert page is not None
|
||||
|
||||
# Whitespace only
|
||||
page = Selector(" \n\t ")
|
||||
assert page is not None
|
||||
|
||||
# Null bytes
|
||||
page = Selector("Hello\x00World")
|
||||
assert "Hello" in page.get_all_text()
|
||||
@@ -0,0 +1,336 @@
|
||||
import pytest
|
||||
import json
|
||||
|
||||
from scrapling import Selector
|
||||
from scrapling.core.custom_types import AttributesHandler
|
||||
|
||||
|
||||
class TestAttributesHandler:
|
||||
"""Test AttributesHandler functionality"""
|
||||
|
||||
@pytest.fixture
|
||||
def sample_html(self):
|
||||
return """
|
||||
<html>
|
||||
<body>
|
||||
<div id="main"
|
||||
class="container active"
|
||||
data-config='{"theme": "dark", "version": 2.5}'
|
||||
data-items='[1, 2, 3, 4, 5]'
|
||||
data-invalid-json='{"broken: json}'
|
||||
title="Main Container"
|
||||
style="color: red; background: blue;"
|
||||
data-empty=""
|
||||
data-number="42"
|
||||
data-bool="true"
|
||||
data-url="https://example.com/page?param=value"
|
||||
custom-attr="custom-value"
|
||||
data-nested='{"user": {"name": "John", "age": 30}}'
|
||||
data-encoded="<div>HTML</div>"
|
||||
onclick="handleClick()"
|
||||
data-null="null"
|
||||
data-undefined="undefined">
|
||||
Content
|
||||
</div>
|
||||
<input type="text"
|
||||
name="username"
|
||||
value="test@example.com"
|
||||
placeholder="Enter email"
|
||||
required
|
||||
disabled>
|
||||
<img src="/images/photo.jpg"
|
||||
alt="Photo"
|
||||
width="100"
|
||||
height="100"
|
||||
loading="lazy">
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def attributes(self, sample_html):
|
||||
page = Selector(sample_html)
|
||||
element = page.css("#main")[0]
|
||||
return element.attrib
|
||||
|
||||
def test_basic_attribute_access(self, attributes):
|
||||
"""Test basic attribute access"""
|
||||
# Dict-like access
|
||||
assert attributes["id"] == "main"
|
||||
assert attributes["class"] == "container active"
|
||||
assert attributes["title"] == "Main Container"
|
||||
|
||||
# Key existence
|
||||
assert "id" in attributes
|
||||
assert "nonexistent" not in attributes
|
||||
|
||||
# Get with default
|
||||
assert attributes.get("id") == "main"
|
||||
assert attributes.get("nonexistent") is None
|
||||
assert attributes.get("nonexistent", "default") == "default"
|
||||
|
||||
def test_iteration_methods(self, attributes):
|
||||
"""Test iteration over attributes"""
|
||||
# Keys
|
||||
keys = list(attributes.keys())
|
||||
assert "id" in keys
|
||||
assert "class" in keys
|
||||
assert "data-config" in keys
|
||||
|
||||
# Values
|
||||
values = list(attributes.values())
|
||||
assert "main" in values
|
||||
assert "container active" in values
|
||||
|
||||
# Items
|
||||
items = dict(attributes.items())
|
||||
assert items["id"] == "main"
|
||||
assert items["class"] == "container active"
|
||||
|
||||
# Length
|
||||
assert len(attributes) > 0
|
||||
|
||||
def test_json_parsing(self, attributes):
|
||||
"""Test JSON parsing from attributes"""
|
||||
# Valid JSON object
|
||||
config = attributes["data-config"].json()
|
||||
assert config["theme"] == "dark"
|
||||
assert config["version"] == 2.5
|
||||
|
||||
# Valid JSON array
|
||||
items = attributes["data-items"].json()
|
||||
assert items == [1, 2, 3, 4, 5]
|
||||
|
||||
# Nested JSON
|
||||
nested = attributes["data-nested"].json()
|
||||
assert nested["user"]["name"] == "John"
|
||||
assert nested["user"]["age"] == 30
|
||||
|
||||
# JSON null
|
||||
assert attributes["data-null"].json() is None
|
||||
|
||||
def test_json_error_handling(self, attributes):
|
||||
"""Test JSON parsing error handling"""
|
||||
# Invalid JSON should raise error or return None
|
||||
with pytest.raises((json.JSONDecodeError, AttributeError)):
|
||||
attributes["data-invalid-json"].json()
|
||||
|
||||
# Non-existent attribute
|
||||
with pytest.raises(KeyError):
|
||||
attributes["nonexistent"].json()
|
||||
|
||||
def test_json_string_property(self, attributes):
|
||||
"""Test json_string property"""
|
||||
# Should return JSON representation of all attributes
|
||||
json_string = attributes.json_string
|
||||
assert isinstance(json_string, bytes)
|
||||
|
||||
# Parse it back
|
||||
parsed = json.loads(json_string)
|
||||
assert parsed["id"] == "main"
|
||||
assert parsed["class"] == "container active"
|
||||
|
||||
def test_search_values(self, attributes):
|
||||
"""Test search_values method"""
|
||||
# Exact match
|
||||
results = list(attributes.search_values("main", partial=False))
|
||||
assert len(results) == 1
|
||||
assert "id" in results[0]
|
||||
|
||||
# Partial match
|
||||
results = list(attributes.search_values("container", partial=True))
|
||||
assert len(results) >= 1
|
||||
found_keys = []
|
||||
for result in results:
|
||||
found_keys.extend(result.keys())
|
||||
assert "class" in found_keys or "title" in found_keys
|
||||
|
||||
# Case sensitivity
|
||||
results = list(attributes.search_values("MAIN", partial=False))
|
||||
assert len(results) == 0 # Should be case-sensitive by default
|
||||
|
||||
# Multiple matches
|
||||
results = list(attributes.search_values("2", partial=True))
|
||||
assert len(results) > 1 # Should find multiple attributes
|
||||
|
||||
# No matches
|
||||
results = list(attributes.search_values("nonexistent", partial=False))
|
||||
assert len(results) == 0
|
||||
|
||||
def test_special_attribute_types(self, sample_html):
|
||||
"""Test handling of special attribute types"""
|
||||
page = Selector(sample_html)
|
||||
|
||||
# Boolean attributes
|
||||
input_elem = page.css("input")[0]
|
||||
assert "required" in input_elem.attrib
|
||||
assert "disabled" in input_elem.attrib
|
||||
|
||||
# Empty attributes
|
||||
main_elem = page.css("#main")[0]
|
||||
assert main_elem.attrib["data-empty"] == ""
|
||||
|
||||
# Numeric string attributes
|
||||
assert main_elem.attrib["data-number"] == "42"
|
||||
assert main_elem.attrib["data-bool"] == "true"
|
||||
|
||||
def test_attribute_modification(self, sample_html):
|
||||
"""Test that AttributesHandler is read-only (if applicable)"""
|
||||
page = Selector(sample_html)
|
||||
element = page.css("#main")[0]
|
||||
attrs = element.attrib
|
||||
|
||||
# Test if attributes can be modified
|
||||
# This behavior depends on implementation
|
||||
original_id = attrs["id"]
|
||||
try:
|
||||
attrs["id"] = "new-id"
|
||||
# If modification is allowed
|
||||
assert attrs["id"] == "new-id"
|
||||
# Reset
|
||||
attrs["id"] = original_id
|
||||
except (TypeError, AttributeError):
|
||||
# If modification is not allowed (read-only)
|
||||
assert attrs["id"] == original_id
|
||||
|
||||
def test_string_representation(self, attributes):
|
||||
"""Test string representations"""
|
||||
# __str__
|
||||
str_repr = str(attributes)
|
||||
assert isinstance(str_repr, str)
|
||||
assert "id" in str_repr or "main" in str_repr
|
||||
|
||||
# __repr__
|
||||
repr_str = repr(attributes)
|
||||
assert isinstance(repr_str, str)
|
||||
|
||||
def test_edge_cases(self, sample_html):
|
||||
"""Test edge cases and special scenarios"""
|
||||
page = Selector(sample_html)
|
||||
|
||||
# Element with no attributes
|
||||
page_with_no_attrs = Selector("<div>Content</div>")
|
||||
elem = page_with_no_attrs.css("div")[0]
|
||||
assert len(elem.attrib) == 0
|
||||
assert list(elem.attrib.keys()) == []
|
||||
assert elem.attrib.get("any") is None
|
||||
|
||||
# Element with encoded content
|
||||
main_elem = page.css("#main")[0]
|
||||
encoded = main_elem.attrib["data-encoded"]
|
||||
assert "<" in encoded # Should decode it
|
||||
|
||||
# Style attribute parsing
|
||||
style = main_elem.attrib["style"]
|
||||
assert "color: red" in style
|
||||
assert "background: blue" in style
|
||||
|
||||
def test_url_attribute(self, attributes):
|
||||
"""Test URL attributes"""
|
||||
url = attributes["data-url"]
|
||||
assert url == "https://example.com/page?param=value"
|
||||
|
||||
# Could test URL joining if AttributesHandler supports it
|
||||
# based on the parent element's base URL
|
||||
|
||||
def test_comparison_operations(self, sample_html):
|
||||
"""Test comparison operations if supported"""
|
||||
page = Selector(sample_html)
|
||||
elem1 = page.css("#main")[0]
|
||||
elem2 = page.css("input")[0]
|
||||
|
||||
# Different elements should have different attributes
|
||||
assert elem1.attrib != elem2.attrib
|
||||
|
||||
# The same element should have equal attributes
|
||||
elem1_again = page.css("#main")[0]
|
||||
assert elem1.attrib == elem1_again.attrib
|
||||
|
||||
def test_complex_search_patterns(self, attributes):
|
||||
"""Test complex search patterns"""
|
||||
# Search for JSON-containing attributes
|
||||
json_attrs = []
|
||||
for key, value in attributes.items():
|
||||
try:
|
||||
if isinstance(value, str) and (value.startswith('{') or value.startswith('[')):
|
||||
json.loads(value)
|
||||
json_attrs.append(key)
|
||||
except:
|
||||
pass
|
||||
|
||||
assert "data-config" in json_attrs
|
||||
assert "data-items" in json_attrs
|
||||
assert "data-nested" in json_attrs
|
||||
|
||||
def test_attribute_filtering(self, attributes):
|
||||
"""Test filtering attributes by patterns"""
|
||||
# Get all data-* attributes
|
||||
data_attrs = {k: v for k, v in attributes.items() if k.startswith("data-")}
|
||||
assert len(data_attrs) > 5
|
||||
assert "data-config" in data_attrs
|
||||
assert "data-items" in data_attrs
|
||||
|
||||
# Get all event handler attributes
|
||||
event_attrs = {k: v for k, v in attributes.items() if k.startswith("on")}
|
||||
assert "onclick" in event_attrs
|
||||
|
||||
def test_performance_with_many_attributes(self):
|
||||
"""Test performance with elements having many attributes"""
|
||||
# Create an element with many attributes
|
||||
attrs_list = [f'data-attr{i}="value{i}"' for i in range(100)]
|
||||
html = f'<div id="test" {" ".join(attrs_list)}>Content</div>'
|
||||
|
||||
page = Selector(html)
|
||||
element = page.css("#test")[0]
|
||||
attribs = element.attrib
|
||||
|
||||
# Should handle many attributes efficiently
|
||||
assert len(attribs) == 101 # id + 100 data attributes
|
||||
|
||||
# Search should still work efficiently
|
||||
results = list(attribs.search_values("value50", partial=False))
|
||||
assert len(results) == 1
|
||||
|
||||
def test_unicode_attributes(self):
|
||||
"""Test handling of Unicode in attributes"""
|
||||
html = """
|
||||
<div id="unicode-test"
|
||||
data-emoji="😀🎉"
|
||||
data-chinese="你好世界"
|
||||
data-arabic="مرحبا بالعالم"
|
||||
data-special="café naïve">
|
||||
</div>
|
||||
"""
|
||||
|
||||
page = Selector(html)
|
||||
attrs = page.css("#unicode-test")[0].attrib
|
||||
|
||||
assert attrs["data-emoji"] == "😀🎉"
|
||||
assert attrs["data-chinese"] == "你好世界"
|
||||
assert attrs["data-arabic"] == "مرحبا بالعالم"
|
||||
assert attrs["data-special"] == "café naïve"
|
||||
|
||||
# Search with Unicode
|
||||
results = list(attrs.search_values("你好", partial=True))
|
||||
assert len(results) == 1
|
||||
|
||||
def test_malformed_attributes(self):
|
||||
"""Test handling of malformed attributes"""
|
||||
# Various malformed HTML scenarios
|
||||
test_cases = [
|
||||
'<div id="test" class=>Content</div>', # Empty attribute value
|
||||
'<div id="test" class>Content</div>', # No attribute value
|
||||
'<div id="test" data-"invalid"="value">Content</div>', # Invalid attribute name
|
||||
'<div id=test class=no-quotes>Content</div>', # Unquoted values
|
||||
]
|
||||
|
||||
for html in test_cases:
|
||||
try:
|
||||
page = Selector(html)
|
||||
if page.css("div"):
|
||||
attrs = page.css("div")[0].attrib
|
||||
# Should handle gracefully without crashing
|
||||
assert isinstance(attrs, AttributesHandler)
|
||||
except:
|
||||
# Some malformed HTML might not parse at all
|
||||
pass
|
||||
@@ -0,0 +1,224 @@
|
||||
import re
|
||||
import pytest
|
||||
|
||||
from scrapling import Selector, Selectors
|
||||
from scrapling.core.custom_types import TextHandler, TextHandlers
|
||||
|
||||
|
||||
class TestAdvancedSelectors:
|
||||
"""Test advanced selector functionality"""
|
||||
|
||||
@pytest.fixture
|
||||
def complex_html(self):
|
||||
return """
|
||||
<html>
|
||||
<body>
|
||||
<div class="container" data-test='{"key": "value"}'>
|
||||
<p>First paragraph</p>
|
||||
<!-- Comment -->
|
||||
<p>Second paragraph</p>
|
||||
<![CDATA[Some CDATA content]]>
|
||||
<div class="nested">
|
||||
<span id="special">Special content</span>
|
||||
<span>Regular content</span>
|
||||
</div>
|
||||
<table>
|
||||
<tr><td>Cell 1</td><td>Cell 2</td></tr>
|
||||
<tr><td>Cell 3</td><td>Cell 4</td></tr>
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
def test_comment_and_cdata_handling(self, complex_html):
|
||||
"""Test handling of comments and CDATA"""
|
||||
# With comments/CDATA kept
|
||||
page = Selector(
|
||||
complex_html,
|
||||
keep_comments=True,
|
||||
keep_cdata=True
|
||||
)
|
||||
content = page.body
|
||||
assert "Comment" in content
|
||||
assert "CDATA" in content
|
||||
|
||||
# Without comments/CDATA
|
||||
page = Selector(
|
||||
complex_html,
|
||||
keep_comments=False,
|
||||
keep_cdata=False
|
||||
)
|
||||
content = page.body
|
||||
assert "Comment" not in content
|
||||
|
||||
def test_advanced_xpath_variables(self, complex_html):
|
||||
"""Test XPath with variables"""
|
||||
page = Selector(complex_html)
|
||||
|
||||
# Using XPath variables
|
||||
cells = page.xpath(
|
||||
"//td[text()=$cell_text]",
|
||||
cell_text="Cell 1"
|
||||
)
|
||||
assert len(cells) == 1
|
||||
assert cells[0].text == "Cell 1"
|
||||
|
||||
def test_pseudo_elements(self, complex_html):
|
||||
"""Test CSS pseudo-elements"""
|
||||
page = Selector(complex_html)
|
||||
|
||||
# ::text pseudo-element
|
||||
texts = page.css("p::text")
|
||||
assert len(texts) == 2
|
||||
assert isinstance(texts[0], TextHandler)
|
||||
|
||||
# ::attr() pseudo-element
|
||||
attrs = page.css("div::attr(class)")
|
||||
assert "container" in attrs
|
||||
|
||||
def test_complex_attribute_operations(self, complex_html):
|
||||
"""Test complex attribute handling"""
|
||||
page = Selector(complex_html)
|
||||
container = page.css(".container")[0]
|
||||
|
||||
# JSON in attributes
|
||||
data = container.attrib["data-test"].json()
|
||||
assert data["key"] == "value"
|
||||
|
||||
# Attribute searching
|
||||
matches = list(container.attrib.search_values("container"))
|
||||
assert len(matches) == 1
|
||||
|
||||
def test_url_joining(self):
|
||||
"""Test URL joining functionality"""
|
||||
page = Selector("<html></html>", url="https://example.com/page")
|
||||
|
||||
# Relative URL
|
||||
assert page.urljoin("../other") == "https://example.com/other"
|
||||
assert page.urljoin("/absolute") == "https://example.com/absolute"
|
||||
assert page.urljoin("relative") == "https://example.com/relative"
|
||||
|
||||
def test_find_operations_edge_cases(self, complex_html):
|
||||
"""Test edge cases in find operations"""
|
||||
page = Selector(complex_html)
|
||||
|
||||
# Multiple argument types
|
||||
_ = page.find_all(
|
||||
"span",
|
||||
["div"],
|
||||
{"class": "nested"},
|
||||
lambda e: e.text != ""
|
||||
)
|
||||
|
||||
# Regex pattern matching
|
||||
pattern = re.compile(r"Cell \d+")
|
||||
cells = page.find_all(pattern)
|
||||
assert len(cells) == 4
|
||||
|
||||
def test_text_operations_edge_cases(self, complex_html):
|
||||
"""Test text operation edge cases"""
|
||||
page = Selector(complex_html)
|
||||
|
||||
# get_all_text with a custom separator
|
||||
text = page.get_all_text(separator=" | ", strip=True)
|
||||
assert " | " in text
|
||||
|
||||
# Ignore specific tags
|
||||
text = page.get_all_text(ignore_tags=("table",))
|
||||
assert "Cell" not in text
|
||||
|
||||
# With empty values
|
||||
text = page.get_all_text(valid_values=False)
|
||||
assert text != ""
|
||||
|
||||
|
||||
class TestTextHandlerAdvanced:
|
||||
"""Test advanced TextHandler functionality"""
|
||||
|
||||
def test_text_handler_operations(self):
|
||||
"""Test various TextHandler operations"""
|
||||
text = TextHandler(" Hello World ")
|
||||
|
||||
# All string methods should return TextHandler
|
||||
assert isinstance(text.strip(), TextHandler)
|
||||
assert isinstance(text.upper(), TextHandler)
|
||||
assert isinstance(text.lower(), TextHandler)
|
||||
assert isinstance(text.replace("World", "Python"), TextHandler)
|
||||
|
||||
# Custom methods
|
||||
assert text.clean() == "Hello World"
|
||||
|
||||
# Sorting
|
||||
text2 = TextHandler("dcba")
|
||||
assert text2.sort() == "abcd"
|
||||
|
||||
def test_text_handler_regex(self):
|
||||
"""Test regex operations on TextHandler"""
|
||||
text = TextHandler("Price: $10.99, Sale: $8.99")
|
||||
|
||||
# Basic regex
|
||||
prices = text.re(r"\$[\d.]+")
|
||||
assert len(prices) == 2
|
||||
assert prices[0] == "$10.99"
|
||||
|
||||
# Case insensitive
|
||||
text2 = TextHandler("HELLO hello HeLLo")
|
||||
matches = text2.re(r"hello", case_sensitive=False)
|
||||
assert len(matches) == 3
|
||||
|
||||
# Clean match
|
||||
text3 = TextHandler(" He l lo ")
|
||||
matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False)
|
||||
assert len(matches) == 1
|
||||
|
||||
def test_text_handlers_operations(self):
|
||||
"""Test TextHandlers list operations"""
|
||||
handlers = TextHandlers([
|
||||
TextHandler("First"),
|
||||
TextHandler("Second"),
|
||||
TextHandler("Third")
|
||||
])
|
||||
|
||||
# Slicing should return TextHandlers
|
||||
assert isinstance(handlers[0:2], TextHandlers)
|
||||
|
||||
# Get methods
|
||||
assert handlers.get() == "First"
|
||||
assert handlers.get("default") == "First"
|
||||
assert TextHandlers([]).get("default") == "default"
|
||||
|
||||
|
||||
class TestSelectorsAdvanced:
|
||||
"""Test advanced Selectors functionality"""
|
||||
|
||||
def test_selectors_filtering(self):
|
||||
"""Test filtering operations on Selectors"""
|
||||
html = """
|
||||
<div>
|
||||
<p class="highlight">Important</p>
|
||||
<p>Regular</p>
|
||||
<p class="highlight">Also important</p>
|
||||
</div>
|
||||
"""
|
||||
page = Selector(html)
|
||||
paragraphs = page.css("p")
|
||||
|
||||
# Filter by class
|
||||
highlighted = paragraphs.filter(lambda p: p.has_class("highlight"))
|
||||
assert len(highlighted) == 2
|
||||
|
||||
# Search for a specific element
|
||||
found = paragraphs.search(lambda p: p.text == "Regular")
|
||||
assert found is not None
|
||||
assert found.text == "Regular"
|
||||
|
||||
def test_selectors_properties(self):
|
||||
"""Test Selectors properties"""
|
||||
html = "<div><p>1</p><p>2</p><p>3</p></div>"
|
||||
page = Selector(html)
|
||||
paragraphs = page.css("p")
|
||||
|
||||
assert paragraphs.first.text == "1"
|
||||
assert paragraphs.last.text == "3"
|
||||
assert paragraphs.length == 3
|
||||
Reference in New Issue
Block a user