tests: Multiple changes to tests

- Remove all tests for the old encoding logic
- Stop using the `nopecha` test page to test Cloudflare solver
- Remove useless tests like testing for infinite timeout
- Fixes to make the code compatible with new changes
This commit is contained in:
Karim shoair
2025-09-14 20:24:46 +03:00
parent ce4fc31f2c
commit eae088a07b
7 changed files with 5 additions and 45 deletions
+2 -12
View File
@@ -1,3 +1,4 @@
from playwright._impl._errors import TimeoutError
import pytest
import pytest_httpbin
@@ -23,14 +24,9 @@ class TestStealthyFetcher:
"basic_url": f"{url}/get",
"html_url": f"{url}/html",
"delayed_url": f"{url}/delay/10", # 10 Seconds delay response
"cookies_url": f"{url}/cookies/set/test/value",
"cloudflare_url": "https://nopecha.com/demo/cloudflare", # Interactive turnstile page
"cookies_url": f"{url}/cookies/set/test/value"
}
async def test_cloudflare_fetch(self, fetcher, urls):
"""Test if Cloudflare bypass is working"""
assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200
async def test_basic_fetch(self, fetcher, urls):
"""Test doing a basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls["status_200"])).status == 200
@@ -86,9 +82,3 @@ class TestStealthyFetcher:
**kwargs
)
assert response.status == 200
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
assert (
await fetcher.async_fetch(urls["delayed_url"], timeout=0)
).status == 200
-6
View File
@@ -90,9 +90,3 @@ class TestDynamicFetcherAsync:
with pytest.raises(Exception):
await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
@pytest.mark.asyncio
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
response = await fetcher.async_fetch(urls["delayed_url"], timeout=0)
assert response.status == 200
-4
View File
@@ -77,7 +77,3 @@ class TestStealthyFetcher:
**kwargs
)
assert response.status == 200
def test_infinite_timeout(self, fetcher):
"""Test if infinite timeout breaks the code or not"""
assert fetcher.fetch(self.delayed_url, timeout=0).status == 200
-15
View File
@@ -1,7 +1,6 @@
from unittest.mock import Mock
from scrapling.parser import Selector
from scrapling.engines.toolbelt.custom import ResponseEncoding
from scrapling.engines.toolbelt.convertor import ResponseFactory, Response
@@ -32,20 +31,6 @@ class TestResponseFactory:
assert response.url == "https://example.com"
assert isinstance(response, Response)
def test_response_encoding_edge_cases(self):
"""Test response encoding handling"""
# Test various content types
test_cases = [
(None, "utf-8"),
("", "utf-8"),
("text/html; charset=invalid", "utf-8"),
("application/octet-stream", "utf-8"),
]
for content_type, expected in test_cases:
encoding = ResponseEncoding.get_value(content_type)
assert encoding == expected
def test_response_history_processing(self):
"""Test processing response history"""
# Mock responses with redirects
+1 -7
View File
@@ -1,7 +1,7 @@
import pytest
from pathlib import Path
from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response
from scrapling.engines.toolbelt.custom import StatusText, Response
from scrapling.engines.toolbelt.navigation import (
construct_proxy_dict,
js_bypass_path
@@ -131,12 +131,6 @@ def status_map():
}
def test_parsing_content_type(content_type_map):
"""Test if parsing different types of 'content-type' returns the expected result"""
for header_value, expected_encoding in content_type_map.items():
assert ResponseEncoding.get_value(header_value) == expected_encoding
def test_parsing_response_status(status_map):
"""Test if using different http responses' status codes returns the expected result"""
for status_code, expected_status_text in status_map.items():