freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: Update and improve Camo/Dynamic tests

Browse Source
This commit is contained in:
Karim shoair
2025-08-16 16:02:17 +03:00
parent 6c9a960a44
commit 4c33c47609
4 changed files with 85 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fetchers/async/test_camoufox.py
+32 -54
View File
@@ -27,37 +27,11 @@ class TestStealthyFetcher:
} }
async def test_basic_fetch(self, fetcher, urls): async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses""" """Test doing a basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls["status_200"])).status == 200 assert (await fetcher.async_fetch(urls["status_200"])).status == 200
assert (await fetcher.async_fetch(urls["status_404"])).status == 404 assert (await fetcher.async_fetch(urls["status_404"])).status == 404
assert (await fetcher.async_fetch(urls["status_501"])).status == 501 assert (await fetcher.async_fetch(urls["status_501"])).status == 501
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert (
await fetcher.async_fetch(urls["basic_url"], network_idle=True)
).status == 200
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not"""
assert (
await fetcher.async_fetch(urls["basic_url"], block_images=True)
).status == 200
assert (
await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
).status == 200
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
assert (
await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
).status == 200
assert (
await fetcher.async_fetch(
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
).status == 200
async def test_cookies_loading(self, fetcher, urls): async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls["cookies_url"]) response = await fetcher.async_fetch(urls["cookies_url"])
@@ -65,7 +39,7 @@ class TestStealthyFetcher:
assert cookies == {"test": "value"} assert cookies == {"test": "value"}
async def test_automation(self, fetcher, urls): async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not""" """Test if automation breaks the code or not"""
async def scroll_page(page): async def scroll_page(page):
await page.mouse.wheel(10, 0) await page.mouse.wheel(10, 0)
@@ -74,34 +48,38 @@ class TestStealthyFetcher:
return page return page
assert ( assert (
await fetcher.async_fetch(urls["html_url"], page_action=scroll_page) await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=True)
).status == 200 ).status == 200
async def test_properties(self, fetcher, urls): @pytest.mark.parametrize(
"""Test if different arguments breaks the code or not""" "kwargs",
assert ( [
await fetcher.async_fetch( {"block_webrtc": True, "allow_webgl": True, "disable_ads": False},
urls["html_url"], block_webrtc=True, allow_webgl=True {"block_webrtc": False, "allow_webgl": True, "block_images": True},
) {"block_webrtc": True, "allow_webgl": False, "disable_resources": True},
).status == 200 {"block_images": True, "disable_resources": True, },
{"wait_selector": "h1", "wait_selector_state": "attached"},
assert ( {"wait_selector": "h1", "wait_selector_state": "visible"},
await fetcher.async_fetch( {
urls["html_url"], block_webrtc=False, allow_webgl=True "network_idle": True,
) "wait": 10,
).status == 200 "cookies": [],
"google_search": True,
assert ( "extra_headers": {"ayo": ""},
await fetcher.async_fetch( "os_randomize": True,
urls["html_url"], block_webrtc=True, allow_webgl=False "disable_ads": True,
) "custom_config": {"keep_comments": False, "keep_cdata": False},
).status == 200 "additional_args": {"window": (1920, 1080)},
},
assert ( ],
await fetcher.async_fetch( )
urls["html_url"], extra_headers={"ayo": ""}, os_randomize=True async def test_properties(self, fetcher, urls, kwargs):
) """Test if different arguments break the code or not"""
).status == 200 response = await fetcher.async_fetch(
urls["html_url"],
**kwargs
)
assert response.status == 200
async def test_infinite_timeout(self, fetcher, urls): async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
tests/fetchers/async/test_dynamic.py
+12 -26
View File
@@ -30,29 +30,6 @@ class TestDynamicFetcherAsync:
response = await fetcher.async_fetch(urls["status_200"]) response = await fetcher.async_fetch(urls["status_200"])
assert response.status == 200 assert response.status == 200
@pytest.mark.asyncio
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
response = await fetcher.async_fetch(urls["basic_url"], network_idle=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make the page does not finish loading or not"""
response = await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
response1 = await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
assert response1.status == 200
response2 = await fetcher.async_fetch(
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
assert response2.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cookies_loading(self, fetcher, urls): async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
@@ -77,12 +54,21 @@ class TestDynamicFetcherAsync:
"kwargs", "kwargs",
[ [
{"disable_webgl": True, "hide_canvas": False}, {"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True}, {"disable_webgl": False, "hide_canvas": True, "disable_resources": True},
{"stealth": True}, # causes issues with GitHub Actions {"stealth": True}, # causes issues with GitHub Actions
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{ {
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0" "google_search": True,
"real_chrome": True,
"wait": 10,
"locale": "en-US",
"extra_headers": {"ayo": ""},
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
"cookies": [],
"network_idle": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
}, },
{"extra_headers": {"ayo": ""}},
], ],
) )
@pytest.mark.asyncio @pytest.mark.asyncio
tests/fetchers/sync/test_camoufox.py
+29 -39
View File
@@ -25,30 +25,11 @@ class TestStealthyFetcher:
self.cookies_url = f"{httpbin.url}/cookies/set/test/value" self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher): def test_basic_fetch(self, fetcher):
"""Test doing basic fetch request with multiple statuses""" """Test doing a basic fetch request with multiple statuses"""
assert fetcher.fetch(self.status_200).status == 200 assert fetcher.fetch(self.status_200).status == 200
assert fetcher.fetch(self.status_404).status == 404 assert fetcher.fetch(self.status_404).status == 404
assert fetcher.fetch(self.status_501).status == 501 assert fetcher.fetch(self.status_501).status == 501
def test_networkidle(self, fetcher):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
def test_blocking_resources(self, fetcher):
"""Test if blocking resources make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, block_images=True).status == 200
assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher): def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
response = fetcher.fetch(self.cookies_url) response = fetcher.fetch(self.cookies_url)
@@ -66,26 +47,35 @@ class TestStealthyFetcher:
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200 assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
def test_properties(self, fetcher): @pytest.mark.parametrize(
"""Test if different arguments breaks the code or not""" "kwargs",
assert ( [
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status {"block_webrtc": True, "allow_webgl": True, "disable_ads": False},
== 200 {"block_webrtc": False, "allow_webgl": True, "block_images": True},
) {"block_webrtc": True, "allow_webgl": False, "disable_resources": True},
assert ( {"block_images": True, "disable_resources": True, },
fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status {"wait_selector": "h1", "wait_selector_state": "attached"},
== 200 {"wait_selector": "h1", "wait_selector_state": "visible"},
) {
assert ( "network_idle": True,
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status "wait": 10,
== 200 "cookies": [],
) "google_search": True,
assert ( "extra_headers": {"ayo": ""},
fetcher.fetch( "os_randomize": True,
self.html_url, extra_headers={"ayo": ""}, os_randomize=True "disable_ads": True,
).status "custom_config": {"keep_comments": False, "keep_cdata": False},
== 200 "additional_args": {"window": (1920, 1080)},
},
],
)
def test_properties(self, fetcher, kwargs):
"""Test if different arguments break the code or not"""
response = fetcher.fetch(
self.html_url,
**kwargs
) )
assert response.status == 200
def test_infinite_timeout(self, fetcher): def test_infinite_timeout(self, fetcher):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
tests/fetchers/sync/test_dynamic.py
+12 -23
View File
@@ -1,5 +1,3 @@
import os
import pytest import pytest
import pytest_httpbin import pytest_httpbin
@@ -33,24 +31,6 @@ class TestDynamicFetcher:
# assert fetcher.fetch(self.status_404).status == 404 # assert fetcher.fetch(self.status_404).status == 404
# assert fetcher.fetch(self.status_501).status == 501 # assert fetcher.fetch(self.status_501).status == 501
def test_networkidle(self, fetcher):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
def test_blocking_resources(self, fetcher):
"""Test if blocking resources make the page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher): def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
response = fetcher.fetch(self.cookies_url) response = fetcher.fetch(self.cookies_url)
@@ -72,12 +52,21 @@ class TestDynamicFetcher:
"kwargs", "kwargs",
[ [
{"disable_webgl": True, "hide_canvas": False}, {"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True}, {"disable_webgl": False, "hide_canvas": True, "disable_resources": True},
{"stealth": True}, # causes issues with GitHub Actions {"stealth": True}, # causes issues with GitHub Actions
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{ {
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0" "google_search": True,
"real_chrome": True,
"wait": 10,
"locale": "en-US",
"extra_headers": {"ayo": ""},
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
"cookies": [],
"network_idle": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
}, },
{"extra_headers": {"ayo": ""}},
], ],
) )
def test_properties(self, fetcher, kwargs): def test_properties(self, fetcher, kwargs):