freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fetchers/async/test_camoufox.py
+55 -43
View File
@@ -17,43 +17,51 @@ class TestStealthyFetcher:
def urls(self, httpbin):
url = httpbin.url
return {
'status_200': f'{url}/status/200',
'status_404': f'{url}/status/404',
'status_501': f'{url}/status/501',
'basic_url': f'{url}/get',
'html_url': f'{url}/html',
'delayed_url': f'{url}/delay/10', # 10 Seconds delay response
'cookies_url': f"{url}/cookies/set/test/value"
"status_200": f"{url}/status/200",
"status_404": f"{url}/status/404",
"status_501": f"{url}/status/501",
"basic_url": f"{url}/get",
"html_url": f"{url}/html",
"delayed_url": f"{url}/delay/10", # 10 Seconds delay response
"cookies_url": f"{url}/cookies/set/test/value",
}
async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls['status_200'])).status == 200
assert (await fetcher.async_fetch(urls['status_404'])).status == 404
assert (await fetcher.async_fetch(urls['status_501'])).status == 501
assert (await fetcher.async_fetch(urls["status_200"])).status == 200
assert (await fetcher.async_fetch(urls["status_404"])).status == 404
assert (await fetcher.async_fetch(urls["status_501"])).status == 501
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], network_idle=True)).status == 200
assert (
await fetcher.async_fetch(urls["basic_url"], network_idle=True)
).status == 200
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], block_images=True)).status == 200
assert (await fetcher.async_fetch(urls['basic_url'], disable_resources=True)).status == 200
assert (
await fetcher.async_fetch(urls["basic_url"], block_images=True)
).status == 200
assert (
await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
).status == 200
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['html_url'], wait_selector='h1')).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
wait_selector='h1',
wait_selector_state='visible'
)).status == 200
assert (
await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
).status == 200
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url'])
assert response.cookies == {'test': 'value'}
response = await fetcher.async_fetch(urls["cookies_url"])
assert response.cookies == {"test": "value"}
async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not"""
@@ -64,34 +72,38 @@ class TestStealthyFetcher:
await page.mouse.up()
return page
assert (await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)).status == 200
assert (
await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
).status == 200
async def test_properties(self, fetcher, urls):
"""Test if different arguments breaks the code or not"""
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=True,
allow_webgl=True
)).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], block_webrtc=True, allow_webgl=True
)
).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=False,
allow_webgl=True
)).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], block_webrtc=False, allow_webgl=True
)
).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=True,
allow_webgl=False
)).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], block_webrtc=True, allow_webgl=False
)
).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
extra_headers={'ayo': ''},
os_randomize=True
)).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], extra_headers={"ayo": ""}, os_randomize=True
)
).status == 200
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
assert (await fetcher.async_fetch(urls['delayed_url'], timeout=None)).status == 200
assert (
await fetcher.async_fetch(urls["delayed_url"], timeout=None)
).status == 200
tests/fetchers/async/test_httpx.py
+92 -51
View File
@@ -16,70 +16,111 @@ class TestAsyncFetcher:
@pytest.fixture(scope="class")
def urls(self, httpbin):
return {
'status_200': f'{httpbin.url}/status/200',
'status_404': f'{httpbin.url}/status/404',
'status_501': f'{httpbin.url}/status/501',
'basic_url': f'{httpbin.url}/get',
'post_url': f'{httpbin.url}/post',
'put_url': f'{httpbin.url}/put',
'delete_url': f'{httpbin.url}/delete',
'html_url': f'{httpbin.url}/html'
"status_200": f"{httpbin.url}/status/200",
"status_404": f"{httpbin.url}/status/404",
"status_501": f"{httpbin.url}/status/501",
"basic_url": f"{httpbin.url}/get",
"post_url": f"{httpbin.url}/post",
"put_url": f"{httpbin.url}/put",
"delete_url": f"{httpbin.url}/delete",
"html_url": f"{httpbin.url}/html",
}
async def test_basic_get(self, fetcher, urls):
"""Test doing basic get request with multiple statuses"""
assert (await fetcher.get(urls['status_200'])).status == 200
assert (await fetcher.get(urls['status_404'])).status == 404
assert (await fetcher.get(urls['status_501'])).status == 501
assert (await fetcher.get(urls["status_200"])).status == 200
assert (await fetcher.get(urls["status_404"])).status == 404
assert (await fetcher.get(urls["status_501"])).status == 501
async def test_get_properties(self, fetcher, urls):
"""Test if different arguments with GET request breaks the code or not"""
assert (await fetcher.get(urls['status_200'], stealthy_headers=True)).status == 200
assert (await fetcher.get(urls['status_200'], follow_redirects=True)).status == 200
assert (await fetcher.get(urls['status_200'], timeout=None)).status == 200
assert (await fetcher.get(
urls['status_200'],
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
assert (
await fetcher.get(urls["status_200"], stealthy_headers=True)
).status == 200
assert (
await fetcher.get(urls["status_200"], follow_redirects=True)
).status == 200
assert (await fetcher.get(urls["status_200"], timeout=None)).status == 200
assert (
await fetcher.get(
urls["status_200"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_post_properties(self, fetcher, urls):
"""Test if different arguments with POST request breaks the code or not"""
assert (await fetcher.post(urls['post_url'], data={'key': 'value'})).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, stealthy_headers=True)).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, follow_redirects=True)).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, timeout=None)).status == 200
assert (await fetcher.post(
urls['post_url'],
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
assert (
await fetcher.post(urls["post_url"], data={"key": "value"})
).status == 200
assert (
await fetcher.post(
urls["post_url"], data={"key": "value"}, stealthy_headers=True
)
).status == 200
assert (
await fetcher.post(
urls["post_url"], data={"key": "value"}, follow_redirects=True
)
).status == 200
assert (
await fetcher.post(urls["post_url"], data={"key": "value"}, timeout=None)
).status == 200
assert (
await fetcher.post(
urls["post_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_put_properties(self, fetcher, urls):
"""Test if different arguments with PUT request breaks the code or not"""
assert (await fetcher.put(urls['put_url'], data={'key': 'value'})).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, stealthy_headers=True)).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, follow_redirects=True)).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, timeout=None)).status in [200, 405]
assert (await fetcher.put(
urls['put_url'],
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status in [200, 405]
assert (await fetcher.put(urls["put_url"], data={"key": "value"})).status in [
200,
405,
]
assert (
await fetcher.put(
urls["put_url"], data={"key": "value"}, stealthy_headers=True
)
).status in [200, 405]
assert (
await fetcher.put(
urls["put_url"], data={"key": "value"}, follow_redirects=True
)
).status in [200, 405]
assert (
await fetcher.put(urls["put_url"], data={"key": "value"}, timeout=None)
).status in [200, 405]
assert (
await fetcher.put(
urls["put_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status in [200, 405]
async def test_delete_properties(self, fetcher, urls):
"""Test if different arguments with DELETE request breaks the code or not"""
assert (await fetcher.delete(urls['delete_url'], stealthy_headers=True)).status == 200
assert (await fetcher.delete(urls['delete_url'], follow_redirects=True)).status == 200
assert (await fetcher.delete(urls['delete_url'], timeout=None)).status == 200
assert (await fetcher.delete(
urls['delete_url'],
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
assert (
await fetcher.delete(urls["delete_url"], stealthy_headers=True)
).status == 200
assert (
await fetcher.delete(urls["delete_url"], follow_redirects=True)
).status == 200
assert (await fetcher.delete(urls["delete_url"], timeout=None)).status == 200
assert (
await fetcher.delete(
urls["delete_url"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
tests/fetchers/async/test_playwright.py
+37 -27
View File
@@ -15,87 +15,97 @@ class TestPlayWrightFetcherAsync:
@pytest.fixture
def urls(self, httpbin):
return {
'status_200': f'{httpbin.url}/status/200',
'status_404': f'{httpbin.url}/status/404',
'status_501': f'{httpbin.url}/status/501',
'basic_url': f'{httpbin.url}/get',
'html_url': f'{httpbin.url}/html',
'delayed_url': f'{httpbin.url}/delay/10',
'cookies_url': f"{httpbin.url}/cookies/set/test/value"
"status_200": f"{httpbin.url}/status/200",
"status_404": f"{httpbin.url}/status/404",
"status_501": f"{httpbin.url}/status/501",
"basic_url": f"{httpbin.url}/get",
"html_url": f"{httpbin.url}/html",
"delayed_url": f"{httpbin.url}/delay/10",
"cookies_url": f"{httpbin.url}/cookies/set/test/value",
}
@pytest.mark.asyncio
async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses"""
response = await fetcher.async_fetch(urls['status_200'])
response = await fetcher.async_fetch(urls["status_200"])
assert response.status == 200
@pytest.mark.asyncio
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], network_idle=True)
response = await fetcher.async_fetch(urls["basic_url"], network_idle=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], disable_resources=True)
response = await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
response1 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1')
response1 = await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
assert response1.status == 200
response2 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1', wait_selector_state='visible')
response2 = await fetcher.async_fetch(
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
assert response2.status == 200
@pytest.mark.asyncio
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url'])
assert response.cookies == {'test': 'value'}
response = await fetcher.async_fetch(urls["cookies_url"])
assert response.cookies == {"test": "value"}
@pytest.mark.asyncio
async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not"""
async def scroll_page(page):
await page.mouse.wheel(10, 0)
await page.mouse.move(100, 400)
await page.mouse.up()
return page
response = await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)
response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
assert response.status == 200
@pytest.mark.parametrize("kwargs", [
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
{"extra_headers": {'ayo': ''}}
])
@pytest.mark.parametrize(
"kwargs",
[
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
},
{"extra_headers": {"ayo": ""}},
],
)
@pytest.mark.asyncio
async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments breaks the code or not"""
response = await fetcher.async_fetch(urls['html_url'], **kwargs)
response = await fetcher.async_fetch(urls["html_url"], **kwargs)
assert response.status == 200
@pytest.mark.asyncio
async def test_cdp_url_invalid(self, fetcher, urls):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah')
await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah', nstbrowser_mode=True)
await fetcher.async_fetch(
urls["html_url"], cdp_url="blahblah", nstbrowser_mode=True
)
with pytest.raises(Exception):
await fetcher.async_fetch(urls['html_url'], cdp_url='ws://blahblah')
await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
@pytest.mark.asyncio
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
response = await fetcher.async_fetch(urls['delayed_url'], timeout=None)
response = await fetcher.async_fetch(urls["delayed_url"], timeout=None)
assert response.status == 200
tests/fetchers/sync/test_camoufox.py
+33 -13
View File
@@ -16,12 +16,12 @@ class TestStealthyFetcher:
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.html_url = f'{httpbin.url}/html'
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher):
@@ -41,15 +41,21 @@ class TestStealthyFetcher:
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
def test_automation(self, fetcher):
"""Test if automation break the code or not"""
def scroll_page(page):
page.mouse.wheel(10, 0)
page.mouse.move(100, 400)
@@ -60,10 +66,24 @@ class TestStealthyFetcher:
def test_properties(self, fetcher):
"""Test if different arguments breaks the code or not"""
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status == 200
assert fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status == 200
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status == 200
assert fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status == 200
assert (
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status
== 200
)
assert (
fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status
== 200
)
assert (
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status
== 200
)
assert (
fetcher.fetch(
self.html_url, extra_headers={"ayo": ""}, os_randomize=True
).status
== 200
)
def test_infinite_timeout(self, fetcher):
"""Test if infinite timeout breaks the code or not"""
tests/fetchers/sync/test_httpx.py
+79 -42
View File
@@ -16,14 +16,14 @@ class TestFetcher:
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.post_url = f'{httpbin.url}/post'
self.put_url = f'{httpbin.url}/put'
self.delete_url = f'{httpbin.url}/delete'
self.html_url = f'{httpbin.url}/html'
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.post_url = f"{httpbin.url}/post"
self.put_url = f"{httpbin.url}/put"
self.delete_url = f"{httpbin.url}/delete"
self.html_url = f"{httpbin.url}/html"
def test_basic_get(self, fetcher):
"""Test doing basic get request with multiple statuses"""
@@ -36,49 +36,86 @@ class TestFetcher:
assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
assert fetcher.get(self.status_200, follow_redirects=True).status == 200
assert fetcher.get(self.status_200, timeout=None).status == 200
assert fetcher.get(
self.status_200,
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
assert (
fetcher.get(
self.status_200,
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_post_properties(self, fetcher):
"""Test if different arguments with POST request breaks the code or not"""
assert fetcher.post(self.post_url, data={'key': 'value'}).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status == 200
assert fetcher.post(
self.post_url,
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
assert fetcher.post(self.post_url, data={"key": "value"}).status == 200
assert (
fetcher.post(
self.post_url, data={"key": "value"}, stealthy_headers=True
).status
== 200
)
assert (
fetcher.post(
self.post_url, data={"key": "value"}, follow_redirects=True
).status
== 200
)
assert (
fetcher.post(self.post_url, data={"key": "value"}, timeout=None).status
== 200
)
assert (
fetcher.post(
self.post_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_put_properties(self, fetcher):
"""Test if different arguments with PUT request breaks the code or not"""
assert fetcher.put(self.put_url, data={'key': 'value'}).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status == 200
assert fetcher.put(
self.put_url,
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
assert fetcher.put(self.put_url, data={"key": "value"}).status == 200
assert (
fetcher.put(
self.put_url, data={"key": "value"}, stealthy_headers=True
).status
== 200
)
assert (
fetcher.put(
self.put_url, data={"key": "value"}, follow_redirects=True
).status
== 200
)
assert (
fetcher.put(self.put_url, data={"key": "value"}, timeout=None).status == 200
)
assert (
fetcher.put(
self.put_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_delete_properties(self, fetcher):
"""Test if different arguments with DELETE request breaks the code or not"""
assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
assert fetcher.delete(self.delete_url, timeout=None).status == 200
assert fetcher.delete(
self.delete_url,
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
assert (
fetcher.delete(
self.delete_url,
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
tests/fetchers/sync/test_playwright.py
+33 -21
View File
@@ -8,7 +8,6 @@ PlayWrightFetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin
class TestPlayWrightFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a StealthyFetcher instance for the entire test class"""
@@ -17,12 +16,12 @@ class TestPlayWrightFetcher:
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.html_url = f'{httpbin.url}/html'
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher):
@@ -42,12 +41,17 @@ class TestPlayWrightFetcher:
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
def test_automation(self, fetcher):
"""Test if automation break the code or not"""
@@ -60,13 +64,18 @@ class TestPlayWrightFetcher:
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
@pytest.mark.parametrize("kwargs", [
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
{"extra_headers": {'ayo': ''}}
])
@pytest.mark.parametrize(
"kwargs",
[
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
},
{"extra_headers": {"ayo": ""}},
],
)
def test_properties(self, fetcher, kwargs):
"""Test if different arguments breaks the code or not"""
response = fetcher.fetch(self.html_url, **kwargs)
@@ -75,15 +84,18 @@ class TestPlayWrightFetcher:
def test_cdp_url_invalid(self, fetcher):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah')
fetcher.fetch(self.html_url, cdp_url="blahblah")
with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True)
fetcher.fetch(self.html_url, cdp_url="blahblah", nstbrowser_mode=True)
with pytest.raises(Exception):
fetcher.fetch(self.html_url, cdp_url='ws://blahblah')
fetcher.fetch(self.html_url, cdp_url="ws://blahblah")
def test_infinite_timeout(self, fetcher, ):
def test_infinite_timeout(
self,
fetcher,
):
"""Test if infinite timeout breaks the code or not"""
response = fetcher.fetch(self.delayed_url, timeout=None)
assert response.status == 200
tests/fetchers/test_utils.py
+105 -64
View File
@@ -7,76 +7,117 @@ from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
def content_type_map():
return {
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome
'text/html; charset=UTF-8': 'UTF-8',
'text/html; charset=ISO-8859-1': 'ISO-8859-1',
'text/html': 'ISO-8859-1',
'application/json; charset=UTF-8': 'UTF-8',
'application/json': 'utf-8',
'text/json': 'utf-8',
'application/javascript; charset=UTF-8': 'UTF-8',
'application/javascript': 'utf-8',
'text/plain; charset=UTF-8': 'UTF-8',
'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
'text/plain': 'ISO-8859-1',
'application/xhtml+xml; charset=UTF-8': 'UTF-8',
'application/xhtml+xml': 'utf-8',
'text/html; charset=windows-1252': 'windows-1252',
'application/json; charset=windows-1252': 'windows-1252',
'text/plain; charset=windows-1252': 'windows-1252',
'text/html; charset="UTF-8"': 'UTF-8',
'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
'text/html; charset="windows-1252"': 'windows-1252',
'application/json; charset="UTF-8"': 'UTF-8',
'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
'application/json; charset="windows-1252"': 'windows-1252',
'text/json; charset="UTF-8"': 'UTF-8',
'application/javascript; charset="UTF-8"': 'UTF-8',
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
'text/plain; charset="UTF-8"': 'UTF-8',
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
'text/plain; charset="windows-1252"': 'windows-1252',
'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
'text/html; charset="US-ASCII"': 'US-ASCII',
'application/json; charset="US-ASCII"': 'US-ASCII',
'text/plain; charset="US-ASCII"': 'US-ASCII',
'text/html; charset="Shift_JIS"': 'Shift_JIS',
'application/json; charset="Shift_JIS"': 'Shift_JIS',
'text/plain; charset="Shift_JIS"': 'Shift_JIS',
'application/xml; charset="UTF-8"': 'UTF-8',
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
'application/xml': 'utf-8',
'text/xml; charset="UTF-8"': 'UTF-8',
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
'text/xml': 'utf-8'
"text/html; charset=UTF-8": "UTF-8",
"text/html; charset=ISO-8859-1": "ISO-8859-1",
"text/html": "ISO-8859-1",
"application/json; charset=UTF-8": "UTF-8",
"application/json": "utf-8",
"text/json": "utf-8",
"application/javascript; charset=UTF-8": "UTF-8",
"application/javascript": "utf-8",
"text/plain; charset=UTF-8": "UTF-8",
"text/plain; charset=ISO-8859-1": "ISO-8859-1",
"text/plain": "ISO-8859-1",
"application/xhtml+xml; charset=UTF-8": "UTF-8",
"application/xhtml+xml": "utf-8",
"text/html; charset=windows-1252": "windows-1252",
"application/json; charset=windows-1252": "windows-1252",
"text/plain; charset=windows-1252": "windows-1252",
'text/html; charset="UTF-8"': "UTF-8",
'text/html; charset="ISO-8859-1"': "ISO-8859-1",
'text/html; charset="windows-1252"': "windows-1252",
'application/json; charset="UTF-8"': "UTF-8",
'application/json; charset="ISO-8859-1"': "ISO-8859-1",
'application/json; charset="windows-1252"': "windows-1252",
'text/json; charset="UTF-8"': "UTF-8",
'application/javascript; charset="UTF-8"': "UTF-8",
'application/javascript; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="UTF-8"': "UTF-8",
'text/plain; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="windows-1252"': "windows-1252",
'application/xhtml+xml; charset="UTF-8"': "UTF-8",
'application/xhtml+xml; charset="ISO-8859-1"': "ISO-8859-1",
'application/xhtml+xml; charset="windows-1252"': "windows-1252",
'text/html; charset="US-ASCII"': "US-ASCII",
'application/json; charset="US-ASCII"': "US-ASCII",
'text/plain; charset="US-ASCII"': "US-ASCII",
'text/html; charset="Shift_JIS"': "Shift_JIS",
'application/json; charset="Shift_JIS"': "Shift_JIS",
'text/plain; charset="Shift_JIS"': "Shift_JIS",
'application/xml; charset="UTF-8"': "UTF-8",
'application/xml; charset="ISO-8859-1"': "ISO-8859-1",
"application/xml": "utf-8",
'text/xml; charset="UTF-8"': "UTF-8",
'text/xml; charset="ISO-8859-1"': "ISO-8859-1",
"text/xml": "utf-8",
}
@pytest.fixture
def status_map():
return {
100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
410: "Gone", 411: "Length Required", 412: "Precondition Failed",
413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
428: "Precondition Required", 429: "Too Many Requests",
431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
503: "Service Unavailable", 504: "Gateway Timeout",
505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
511: "Network Authentication Required"
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
}