chore: migrating to ruff and updating pre-commit hooks
This commit is contained in:
Karim shoair
@@ -17,43 +17,51 @@ class TestStealthyFetcher:
|
||||
def urls(self, httpbin):
|
||||
url = httpbin.url
|
||||
return {
|
||||
'status_200': f'{url}/status/200',
|
||||
'status_404': f'{url}/status/404',
|
||||
'status_501': f'{url}/status/501',
|
||||
'basic_url': f'{url}/get',
|
||||
'html_url': f'{url}/html',
|
||||
'delayed_url': f'{url}/delay/10', # 10 Seconds delay response
|
||||
'cookies_url': f"{url}/cookies/set/test/value"
|
||||
"status_200": f"{url}/status/200",
|
||||
"status_404": f"{url}/status/404",
|
||||
"status_501": f"{url}/status/501",
|
||||
"basic_url": f"{url}/get",
|
||||
"html_url": f"{url}/html",
|
||||
"delayed_url": f"{url}/delay/10", # 10 Seconds delay response
|
||||
"cookies_url": f"{url}/cookies/set/test/value",
|
||||
}
|
||||
|
||||
async def test_basic_fetch(self, fetcher, urls):
|
||||
"""Test doing basic fetch request with multiple statuses"""
|
||||
assert (await fetcher.async_fetch(urls['status_200'])).status == 200
|
||||
assert (await fetcher.async_fetch(urls['status_404'])).status == 404
|
||||
assert (await fetcher.async_fetch(urls['status_501'])).status == 501
|
||||
assert (await fetcher.async_fetch(urls["status_200"])).status == 200
|
||||
assert (await fetcher.async_fetch(urls["status_404"])).status == 404
|
||||
assert (await fetcher.async_fetch(urls["status_501"])).status == 501
|
||||
|
||||
async def test_networkidle(self, fetcher, urls):
|
||||
"""Test if waiting for `networkidle` make page does not finish loading or not"""
|
||||
assert (await fetcher.async_fetch(urls['basic_url'], network_idle=True)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["basic_url"], network_idle=True)
|
||||
).status == 200
|
||||
|
||||
async def test_blocking_resources(self, fetcher, urls):
|
||||
"""Test if blocking resources make page does not finish loading or not"""
|
||||
assert (await fetcher.async_fetch(urls['basic_url'], block_images=True)).status == 200
|
||||
assert (await fetcher.async_fetch(urls['basic_url'], disable_resources=True)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["basic_url"], block_images=True)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
|
||||
).status == 200
|
||||
|
||||
async def test_waiting_selector(self, fetcher, urls):
|
||||
"""Test if waiting for a selector make page does not finish loading or not"""
|
||||
assert (await fetcher.async_fetch(urls['html_url'], wait_selector='h1')).status == 200
|
||||
assert (await fetcher.async_fetch(
|
||||
urls['html_url'],
|
||||
wait_selector='h1',
|
||||
wait_selector_state='visible'
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
|
||||
)
|
||||
).status == 200
|
||||
|
||||
async def test_cookies_loading(self, fetcher, urls):
|
||||
"""Test if cookies are set after the request"""
|
||||
response = await fetcher.async_fetch(urls['cookies_url'])
|
||||
assert response.cookies == {'test': 'value'}
|
||||
response = await fetcher.async_fetch(urls["cookies_url"])
|
||||
assert response.cookies == {"test": "value"}
|
||||
|
||||
async def test_automation(self, fetcher, urls):
|
||||
"""Test if automation break the code or not"""
|
||||
@@ -64,34 +72,38 @@ class TestStealthyFetcher:
|
||||
await page.mouse.up()
|
||||
return page
|
||||
|
||||
assert (await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
|
||||
).status == 200
|
||||
|
||||
async def test_properties(self, fetcher, urls):
|
||||
"""Test if different arguments breaks the code or not"""
|
||||
assert (await fetcher.async_fetch(
|
||||
urls['html_url'],
|
||||
block_webrtc=True,
|
||||
allow_webgl=True
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], block_webrtc=True, allow_webgl=True
|
||||
)
|
||||
).status == 200
|
||||
|
||||
assert (await fetcher.async_fetch(
|
||||
urls['html_url'],
|
||||
block_webrtc=False,
|
||||
allow_webgl=True
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], block_webrtc=False, allow_webgl=True
|
||||
)
|
||||
).status == 200
|
||||
|
||||
assert (await fetcher.async_fetch(
|
||||
urls['html_url'],
|
||||
block_webrtc=True,
|
||||
allow_webgl=False
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], block_webrtc=True, allow_webgl=False
|
||||
)
|
||||
).status == 200
|
||||
|
||||
assert (await fetcher.async_fetch(
|
||||
urls['html_url'],
|
||||
extra_headers={'ayo': ''},
|
||||
os_randomize=True
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], extra_headers={"ayo": ""}, os_randomize=True
|
||||
)
|
||||
).status == 200
|
||||
|
||||
async def test_infinite_timeout(self, fetcher, urls):
|
||||
"""Test if infinite timeout breaks the code or not"""
|
||||
assert (await fetcher.async_fetch(urls['delayed_url'], timeout=None)).status == 200
|
||||
assert (
|
||||
await fetcher.async_fetch(urls["delayed_url"], timeout=None)
|
||||
).status == 200
|
||||
|
||||
@@ -16,70 +16,111 @@ class TestAsyncFetcher:
|
||||
@pytest.fixture(scope="class")
|
||||
def urls(self, httpbin):
|
||||
return {
|
||||
'status_200': f'{httpbin.url}/status/200',
|
||||
'status_404': f'{httpbin.url}/status/404',
|
||||
'status_501': f'{httpbin.url}/status/501',
|
||||
'basic_url': f'{httpbin.url}/get',
|
||||
'post_url': f'{httpbin.url}/post',
|
||||
'put_url': f'{httpbin.url}/put',
|
||||
'delete_url': f'{httpbin.url}/delete',
|
||||
'html_url': f'{httpbin.url}/html'
|
||||
"status_200": f"{httpbin.url}/status/200",
|
||||
"status_404": f"{httpbin.url}/status/404",
|
||||
"status_501": f"{httpbin.url}/status/501",
|
||||
"basic_url": f"{httpbin.url}/get",
|
||||
"post_url": f"{httpbin.url}/post",
|
||||
"put_url": f"{httpbin.url}/put",
|
||||
"delete_url": f"{httpbin.url}/delete",
|
||||
"html_url": f"{httpbin.url}/html",
|
||||
}
|
||||
|
||||
async def test_basic_get(self, fetcher, urls):
|
||||
"""Test doing basic get request with multiple statuses"""
|
||||
assert (await fetcher.get(urls['status_200'])).status == 200
|
||||
assert (await fetcher.get(urls['status_404'])).status == 404
|
||||
assert (await fetcher.get(urls['status_501'])).status == 501
|
||||
assert (await fetcher.get(urls["status_200"])).status == 200
|
||||
assert (await fetcher.get(urls["status_404"])).status == 404
|
||||
assert (await fetcher.get(urls["status_501"])).status == 501
|
||||
|
||||
async def test_get_properties(self, fetcher, urls):
|
||||
"""Test if different arguments with GET request breaks the code or not"""
|
||||
assert (await fetcher.get(urls['status_200'], stealthy_headers=True)).status == 200
|
||||
assert (await fetcher.get(urls['status_200'], follow_redirects=True)).status == 200
|
||||
assert (await fetcher.get(urls['status_200'], timeout=None)).status == 200
|
||||
assert (await fetcher.get(
|
||||
urls['status_200'],
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.get(urls["status_200"], stealthy_headers=True)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.get(urls["status_200"], follow_redirects=True)
|
||||
).status == 200
|
||||
assert (await fetcher.get(urls["status_200"], timeout=None)).status == 200
|
||||
assert (
|
||||
await fetcher.get(
|
||||
urls["status_200"],
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
)
|
||||
).status == 200
|
||||
|
||||
async def test_post_properties(self, fetcher, urls):
|
||||
"""Test if different arguments with POST request breaks the code or not"""
|
||||
assert (await fetcher.post(urls['post_url'], data={'key': 'value'})).status == 200
|
||||
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, stealthy_headers=True)).status == 200
|
||||
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, follow_redirects=True)).status == 200
|
||||
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, timeout=None)).status == 200
|
||||
assert (await fetcher.post(
|
||||
urls['post_url'],
|
||||
data={'key': 'value'},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.post(urls["post_url"], data={"key": "value"})
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.post(
|
||||
urls["post_url"], data={"key": "value"}, stealthy_headers=True
|
||||
)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.post(
|
||||
urls["post_url"], data={"key": "value"}, follow_redirects=True
|
||||
)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.post(urls["post_url"], data={"key": "value"}, timeout=None)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.post(
|
||||
urls["post_url"],
|
||||
data={"key": "value"},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
)
|
||||
).status == 200
|
||||
|
||||
async def test_put_properties(self, fetcher, urls):
|
||||
"""Test if different arguments with PUT request breaks the code or not"""
|
||||
assert (await fetcher.put(urls['put_url'], data={'key': 'value'})).status in [200, 405]
|
||||
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, stealthy_headers=True)).status in [200, 405]
|
||||
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, follow_redirects=True)).status in [200, 405]
|
||||
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, timeout=None)).status in [200, 405]
|
||||
assert (await fetcher.put(
|
||||
urls['put_url'],
|
||||
data={'key': 'value'},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
)).status in [200, 405]
|
||||
assert (await fetcher.put(urls["put_url"], data={"key": "value"})).status in [
|
||||
200,
|
||||
405,
|
||||
]
|
||||
assert (
|
||||
await fetcher.put(
|
||||
urls["put_url"], data={"key": "value"}, stealthy_headers=True
|
||||
)
|
||||
).status in [200, 405]
|
||||
assert (
|
||||
await fetcher.put(
|
||||
urls["put_url"], data={"key": "value"}, follow_redirects=True
|
||||
)
|
||||
).status in [200, 405]
|
||||
assert (
|
||||
await fetcher.put(urls["put_url"], data={"key": "value"}, timeout=None)
|
||||
).status in [200, 405]
|
||||
assert (
|
||||
await fetcher.put(
|
||||
urls["put_url"],
|
||||
data={"key": "value"},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
)
|
||||
).status in [200, 405]
|
||||
|
||||
async def test_delete_properties(self, fetcher, urls):
|
||||
"""Test if different arguments with DELETE request breaks the code or not"""
|
||||
assert (await fetcher.delete(urls['delete_url'], stealthy_headers=True)).status == 200
|
||||
assert (await fetcher.delete(urls['delete_url'], follow_redirects=True)).status == 200
|
||||
assert (await fetcher.delete(urls['delete_url'], timeout=None)).status == 200
|
||||
assert (await fetcher.delete(
|
||||
urls['delete_url'],
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
)).status == 200
|
||||
assert (
|
||||
await fetcher.delete(urls["delete_url"], stealthy_headers=True)
|
||||
).status == 200
|
||||
assert (
|
||||
await fetcher.delete(urls["delete_url"], follow_redirects=True)
|
||||
).status == 200
|
||||
assert (await fetcher.delete(urls["delete_url"], timeout=None)).status == 200
|
||||
assert (
|
||||
await fetcher.delete(
|
||||
urls["delete_url"],
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
)
|
||||
).status == 200
|
||||
|
||||
@@ -15,87 +15,97 @@ class TestPlayWrightFetcherAsync:
|
||||
@pytest.fixture
|
||||
def urls(self, httpbin):
|
||||
return {
|
||||
'status_200': f'{httpbin.url}/status/200',
|
||||
'status_404': f'{httpbin.url}/status/404',
|
||||
'status_501': f'{httpbin.url}/status/501',
|
||||
'basic_url': f'{httpbin.url}/get',
|
||||
'html_url': f'{httpbin.url}/html',
|
||||
'delayed_url': f'{httpbin.url}/delay/10',
|
||||
'cookies_url': f"{httpbin.url}/cookies/set/test/value"
|
||||
"status_200": f"{httpbin.url}/status/200",
|
||||
"status_404": f"{httpbin.url}/status/404",
|
||||
"status_501": f"{httpbin.url}/status/501",
|
||||
"basic_url": f"{httpbin.url}/get",
|
||||
"html_url": f"{httpbin.url}/html",
|
||||
"delayed_url": f"{httpbin.url}/delay/10",
|
||||
"cookies_url": f"{httpbin.url}/cookies/set/test/value",
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_fetch(self, fetcher, urls):
|
||||
"""Test doing basic fetch request with multiple statuses"""
|
||||
response = await fetcher.async_fetch(urls['status_200'])
|
||||
response = await fetcher.async_fetch(urls["status_200"])
|
||||
assert response.status == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_networkidle(self, fetcher, urls):
|
||||
"""Test if waiting for `networkidle` make page does not finish loading or not"""
|
||||
response = await fetcher.async_fetch(urls['basic_url'], network_idle=True)
|
||||
response = await fetcher.async_fetch(urls["basic_url"], network_idle=True)
|
||||
assert response.status == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blocking_resources(self, fetcher, urls):
|
||||
"""Test if blocking resources make page does not finish loading or not"""
|
||||
response = await fetcher.async_fetch(urls['basic_url'], disable_resources=True)
|
||||
response = await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
|
||||
assert response.status == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_waiting_selector(self, fetcher, urls):
|
||||
"""Test if waiting for a selector make page does not finish loading or not"""
|
||||
response1 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1')
|
||||
response1 = await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
|
||||
assert response1.status == 200
|
||||
|
||||
response2 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1', wait_selector_state='visible')
|
||||
response2 = await fetcher.async_fetch(
|
||||
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
|
||||
)
|
||||
assert response2.status == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookies_loading(self, fetcher, urls):
|
||||
"""Test if cookies are set after the request"""
|
||||
response = await fetcher.async_fetch(urls['cookies_url'])
|
||||
assert response.cookies == {'test': 'value'}
|
||||
response = await fetcher.async_fetch(urls["cookies_url"])
|
||||
assert response.cookies == {"test": "value"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_automation(self, fetcher, urls):
|
||||
"""Test if automation break the code or not"""
|
||||
|
||||
async def scroll_page(page):
|
||||
await page.mouse.wheel(10, 0)
|
||||
await page.mouse.move(100, 400)
|
||||
await page.mouse.up()
|
||||
return page
|
||||
|
||||
response = await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)
|
||||
response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
|
||||
assert response.status == 200
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [
|
||||
{"disable_webgl": True, "hide_canvas": False},
|
||||
{"disable_webgl": False, "hide_canvas": True},
|
||||
# {"stealth": True}, # causes issues with Github Actions
|
||||
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
|
||||
{"extra_headers": {'ayo': ''}}
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"disable_webgl": True, "hide_canvas": False},
|
||||
{"disable_webgl": False, "hide_canvas": True},
|
||||
# {"stealth": True}, # causes issues with Github Actions
|
||||
{
|
||||
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
|
||||
},
|
||||
{"extra_headers": {"ayo": ""}},
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_properties(self, fetcher, urls, kwargs):
|
||||
"""Test if different arguments breaks the code or not"""
|
||||
response = await fetcher.async_fetch(urls['html_url'], **kwargs)
|
||||
response = await fetcher.async_fetch(urls["html_url"], **kwargs)
|
||||
assert response.status == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cdp_url_invalid(self, fetcher, urls):
|
||||
"""Test if invalid CDP URLs raise appropriate exceptions"""
|
||||
with pytest.raises(ValueError):
|
||||
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah')
|
||||
await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah', nstbrowser_mode=True)
|
||||
await fetcher.async_fetch(
|
||||
urls["html_url"], cdp_url="blahblah", nstbrowser_mode=True
|
||||
)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await fetcher.async_fetch(urls['html_url'], cdp_url='ws://blahblah')
|
||||
await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infinite_timeout(self, fetcher, urls):
|
||||
"""Test if infinite timeout breaks the code or not"""
|
||||
response = await fetcher.async_fetch(urls['delayed_url'], timeout=None)
|
||||
response = await fetcher.async_fetch(urls["delayed_url"], timeout=None)
|
||||
assert response.status == 200
|
||||
|
||||
@@ -16,12 +16,12 @@ class TestStealthyFetcher:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_urls(self, httpbin):
|
||||
"""Fixture to set up URLs for testing"""
|
||||
self.status_200 = f'{httpbin.url}/status/200'
|
||||
self.status_404 = f'{httpbin.url}/status/404'
|
||||
self.status_501 = f'{httpbin.url}/status/501'
|
||||
self.basic_url = f'{httpbin.url}/get'
|
||||
self.html_url = f'{httpbin.url}/html'
|
||||
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
|
||||
self.status_200 = f"{httpbin.url}/status/200"
|
||||
self.status_404 = f"{httpbin.url}/status/404"
|
||||
self.status_501 = f"{httpbin.url}/status/501"
|
||||
self.basic_url = f"{httpbin.url}/get"
|
||||
self.html_url = f"{httpbin.url}/html"
|
||||
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
|
||||
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
|
||||
|
||||
def test_basic_fetch(self, fetcher):
|
||||
@@ -41,15 +41,21 @@ class TestStealthyFetcher:
|
||||
|
||||
def test_waiting_selector(self, fetcher):
|
||||
"""Test if waiting for a selector make page does not finish loading or not"""
|
||||
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
|
||||
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
|
||||
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
|
||||
assert (
|
||||
fetcher.fetch(
|
||||
self.html_url, wait_selector="h1", wait_selector_state="visible"
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_cookies_loading(self, fetcher):
|
||||
"""Test if cookies are set after the request"""
|
||||
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
|
||||
assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
|
||||
|
||||
def test_automation(self, fetcher):
|
||||
"""Test if automation break the code or not"""
|
||||
|
||||
def scroll_page(page):
|
||||
page.mouse.wheel(10, 0)
|
||||
page.mouse.move(100, 400)
|
||||
@@ -60,10 +66,24 @@ class TestStealthyFetcher:
|
||||
|
||||
def test_properties(self, fetcher):
|
||||
"""Test if different arguments breaks the code or not"""
|
||||
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status == 200
|
||||
assert fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status == 200
|
||||
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status == 200
|
||||
assert fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status == 200
|
||||
assert (
|
||||
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.fetch(
|
||||
self.html_url, extra_headers={"ayo": ""}, os_randomize=True
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_infinite_timeout(self, fetcher):
|
||||
"""Test if infinite timeout breaks the code or not"""
|
||||
|
||||
@@ -16,14 +16,14 @@ class TestFetcher:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_urls(self, httpbin):
|
||||
"""Fixture to set up URLs for testing"""
|
||||
self.status_200 = f'{httpbin.url}/status/200'
|
||||
self.status_404 = f'{httpbin.url}/status/404'
|
||||
self.status_501 = f'{httpbin.url}/status/501'
|
||||
self.basic_url = f'{httpbin.url}/get'
|
||||
self.post_url = f'{httpbin.url}/post'
|
||||
self.put_url = f'{httpbin.url}/put'
|
||||
self.delete_url = f'{httpbin.url}/delete'
|
||||
self.html_url = f'{httpbin.url}/html'
|
||||
self.status_200 = f"{httpbin.url}/status/200"
|
||||
self.status_404 = f"{httpbin.url}/status/404"
|
||||
self.status_501 = f"{httpbin.url}/status/501"
|
||||
self.basic_url = f"{httpbin.url}/get"
|
||||
self.post_url = f"{httpbin.url}/post"
|
||||
self.put_url = f"{httpbin.url}/put"
|
||||
self.delete_url = f"{httpbin.url}/delete"
|
||||
self.html_url = f"{httpbin.url}/html"
|
||||
|
||||
def test_basic_get(self, fetcher):
|
||||
"""Test doing basic get request with multiple statuses"""
|
||||
@@ -36,49 +36,86 @@ class TestFetcher:
|
||||
assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
|
||||
assert fetcher.get(self.status_200, follow_redirects=True).status == 200
|
||||
assert fetcher.get(self.status_200, timeout=None).status == 200
|
||||
assert fetcher.get(
|
||||
self.status_200,
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
).status == 200
|
||||
assert (
|
||||
fetcher.get(
|
||||
self.status_200,
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_post_properties(self, fetcher):
|
||||
"""Test if different arguments with POST request breaks the code or not"""
|
||||
assert fetcher.post(self.post_url, data={'key': 'value'}).status == 200
|
||||
assert fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status == 200
|
||||
assert fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status == 200
|
||||
assert fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status == 200
|
||||
assert fetcher.post(
|
||||
self.post_url,
|
||||
data={'key': 'value'},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
).status == 200
|
||||
assert fetcher.post(self.post_url, data={"key": "value"}).status == 200
|
||||
assert (
|
||||
fetcher.post(
|
||||
self.post_url, data={"key": "value"}, stealthy_headers=True
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.post(
|
||||
self.post_url, data={"key": "value"}, follow_redirects=True
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.post(self.post_url, data={"key": "value"}, timeout=None).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.post(
|
||||
self.post_url,
|
||||
data={"key": "value"},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_put_properties(self, fetcher):
|
||||
"""Test if different arguments with PUT request breaks the code or not"""
|
||||
assert fetcher.put(self.put_url, data={'key': 'value'}).status == 200
|
||||
assert fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status == 200
|
||||
assert fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status == 200
|
||||
assert fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status == 200
|
||||
assert fetcher.put(
|
||||
self.put_url,
|
||||
data={'key': 'value'},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
).status == 200
|
||||
assert fetcher.put(self.put_url, data={"key": "value"}).status == 200
|
||||
assert (
|
||||
fetcher.put(
|
||||
self.put_url, data={"key": "value"}, stealthy_headers=True
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.put(
|
||||
self.put_url, data={"key": "value"}, follow_redirects=True
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
fetcher.put(self.put_url, data={"key": "value"}, timeout=None).status == 200
|
||||
)
|
||||
assert (
|
||||
fetcher.put(
|
||||
self.put_url,
|
||||
data={"key": "value"},
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_delete_properties(self, fetcher):
|
||||
"""Test if different arguments with DELETE request breaks the code or not"""
|
||||
assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
|
||||
assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
|
||||
assert fetcher.delete(self.delete_url, timeout=None).status == 200
|
||||
assert fetcher.delete(
|
||||
self.delete_url,
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None
|
||||
).status == 200
|
||||
assert (
|
||||
fetcher.delete(
|
||||
self.delete_url,
|
||||
stealthy_headers=True,
|
||||
follow_redirects=True,
|
||||
timeout=None,
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
@@ -8,7 +8,6 @@ PlayWrightFetcher.auto_match = True
|
||||
|
||||
@pytest_httpbin.use_class_based_httpbin
|
||||
class TestPlayWrightFetcher:
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def fetcher(self):
|
||||
"""Fixture to create a StealthyFetcher instance for the entire test class"""
|
||||
@@ -17,12 +16,12 @@ class TestPlayWrightFetcher:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_urls(self, httpbin):
|
||||
"""Fixture to set up URLs for testing"""
|
||||
self.status_200 = f'{httpbin.url}/status/200'
|
||||
self.status_404 = f'{httpbin.url}/status/404'
|
||||
self.status_501 = f'{httpbin.url}/status/501'
|
||||
self.basic_url = f'{httpbin.url}/get'
|
||||
self.html_url = f'{httpbin.url}/html'
|
||||
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
|
||||
self.status_200 = f"{httpbin.url}/status/200"
|
||||
self.status_404 = f"{httpbin.url}/status/404"
|
||||
self.status_501 = f"{httpbin.url}/status/501"
|
||||
self.basic_url = f"{httpbin.url}/get"
|
||||
self.html_url = f"{httpbin.url}/html"
|
||||
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
|
||||
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
|
||||
|
||||
def test_basic_fetch(self, fetcher):
|
||||
@@ -42,12 +41,17 @@ class TestPlayWrightFetcher:
|
||||
|
||||
def test_waiting_selector(self, fetcher):
|
||||
"""Test if waiting for a selector make page does not finish loading or not"""
|
||||
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
|
||||
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
|
||||
assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
|
||||
assert (
|
||||
fetcher.fetch(
|
||||
self.html_url, wait_selector="h1", wait_selector_state="visible"
|
||||
).status
|
||||
== 200
|
||||
)
|
||||
|
||||
def test_cookies_loading(self, fetcher):
|
||||
"""Test if cookies are set after the request"""
|
||||
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
|
||||
assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
|
||||
|
||||
def test_automation(self, fetcher):
|
||||
"""Test if automation break the code or not"""
|
||||
@@ -60,13 +64,18 @@ class TestPlayWrightFetcher:
|
||||
|
||||
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [
|
||||
{"disable_webgl": True, "hide_canvas": False},
|
||||
{"disable_webgl": False, "hide_canvas": True},
|
||||
# {"stealth": True}, # causes issues with Github Actions
|
||||
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
|
||||
{"extra_headers": {'ayo': ''}}
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"disable_webgl": True, "hide_canvas": False},
|
||||
{"disable_webgl": False, "hide_canvas": True},
|
||||
# {"stealth": True}, # causes issues with Github Actions
|
||||
{
|
||||
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
|
||||
},
|
||||
{"extra_headers": {"ayo": ""}},
|
||||
],
|
||||
)
|
||||
def test_properties(self, fetcher, kwargs):
|
||||
"""Test if different arguments breaks the code or not"""
|
||||
response = fetcher.fetch(self.html_url, **kwargs)
|
||||
@@ -75,15 +84,18 @@ class TestPlayWrightFetcher:
|
||||
def test_cdp_url_invalid(self, fetcher):
|
||||
"""Test if invalid CDP URLs raise appropriate exceptions"""
|
||||
with pytest.raises(ValueError):
|
||||
fetcher.fetch(self.html_url, cdp_url='blahblah')
|
||||
fetcher.fetch(self.html_url, cdp_url="blahblah")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True)
|
||||
fetcher.fetch(self.html_url, cdp_url="blahblah", nstbrowser_mode=True)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
fetcher.fetch(self.html_url, cdp_url='ws://blahblah')
|
||||
fetcher.fetch(self.html_url, cdp_url="ws://blahblah")
|
||||
|
||||
def test_infinite_timeout(self, fetcher, ):
|
||||
def test_infinite_timeout(
|
||||
self,
|
||||
fetcher,
|
||||
):
|
||||
"""Test if infinite timeout breaks the code or not"""
|
||||
response = fetcher.fetch(self.delayed_url, timeout=None)
|
||||
assert response.status == 200
|
||||
|
||||
+105
-64
@@ -7,76 +7,117 @@ from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
|
||||
def content_type_map():
|
||||
return {
|
||||
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome
|
||||
'text/html; charset=UTF-8': 'UTF-8',
|
||||
'text/html; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/html': 'ISO-8859-1',
|
||||
'application/json; charset=UTF-8': 'UTF-8',
|
||||
'application/json': 'utf-8',
|
||||
'text/json': 'utf-8',
|
||||
'application/javascript; charset=UTF-8': 'UTF-8',
|
||||
'application/javascript': 'utf-8',
|
||||
'text/plain; charset=UTF-8': 'UTF-8',
|
||||
'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/plain': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset=UTF-8': 'UTF-8',
|
||||
'application/xhtml+xml': 'utf-8',
|
||||
'text/html; charset=windows-1252': 'windows-1252',
|
||||
'application/json; charset=windows-1252': 'windows-1252',
|
||||
'text/plain; charset=windows-1252': 'windows-1252',
|
||||
'text/html; charset="UTF-8"': 'UTF-8',
|
||||
'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/html; charset="windows-1252"': 'windows-1252',
|
||||
'application/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/json; charset="windows-1252"': 'windows-1252',
|
||||
'text/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="UTF-8"': 'UTF-8',
|
||||
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="windows-1252"': 'windows-1252',
|
||||
'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
|
||||
'text/html; charset="US-ASCII"': 'US-ASCII',
|
||||
'application/json; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/plain; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/html; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/json; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'text/plain; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xml': 'utf-8',
|
||||
'text/xml; charset="UTF-8"': 'UTF-8',
|
||||
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/xml': 'utf-8'
|
||||
"text/html; charset=UTF-8": "UTF-8",
|
||||
"text/html; charset=ISO-8859-1": "ISO-8859-1",
|
||||
"text/html": "ISO-8859-1",
|
||||
"application/json; charset=UTF-8": "UTF-8",
|
||||
"application/json": "utf-8",
|
||||
"text/json": "utf-8",
|
||||
"application/javascript; charset=UTF-8": "UTF-8",
|
||||
"application/javascript": "utf-8",
|
||||
"text/plain; charset=UTF-8": "UTF-8",
|
||||
"text/plain; charset=ISO-8859-1": "ISO-8859-1",
|
||||
"text/plain": "ISO-8859-1",
|
||||
"application/xhtml+xml; charset=UTF-8": "UTF-8",
|
||||
"application/xhtml+xml": "utf-8",
|
||||
"text/html; charset=windows-1252": "windows-1252",
|
||||
"application/json; charset=windows-1252": "windows-1252",
|
||||
"text/plain; charset=windows-1252": "windows-1252",
|
||||
'text/html; charset="UTF-8"': "UTF-8",
|
||||
'text/html; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
'text/html; charset="windows-1252"': "windows-1252",
|
||||
'application/json; charset="UTF-8"': "UTF-8",
|
||||
'application/json; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
'application/json; charset="windows-1252"': "windows-1252",
|
||||
'text/json; charset="UTF-8"': "UTF-8",
|
||||
'application/javascript; charset="UTF-8"': "UTF-8",
|
||||
'application/javascript; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
'text/plain; charset="UTF-8"': "UTF-8",
|
||||
'text/plain; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
'text/plain; charset="windows-1252"': "windows-1252",
|
||||
'application/xhtml+xml; charset="UTF-8"': "UTF-8",
|
||||
'application/xhtml+xml; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
'application/xhtml+xml; charset="windows-1252"': "windows-1252",
|
||||
'text/html; charset="US-ASCII"': "US-ASCII",
|
||||
'application/json; charset="US-ASCII"': "US-ASCII",
|
||||
'text/plain; charset="US-ASCII"': "US-ASCII",
|
||||
'text/html; charset="Shift_JIS"': "Shift_JIS",
|
||||
'application/json; charset="Shift_JIS"': "Shift_JIS",
|
||||
'text/plain; charset="Shift_JIS"': "Shift_JIS",
|
||||
'application/xml; charset="UTF-8"': "UTF-8",
|
||||
'application/xml; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
"application/xml": "utf-8",
|
||||
'text/xml; charset="UTF-8"': "UTF-8",
|
||||
'text/xml; charset="ISO-8859-1"': "ISO-8859-1",
|
||||
"text/xml": "utf-8",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def status_map():
|
||||
return {
|
||||
100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
|
||||
200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
|
||||
204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
|
||||
208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
|
||||
301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
|
||||
305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
|
||||
400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
|
||||
404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
|
||||
407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
|
||||
410: "Gone", 411: "Length Required", 412: "Precondition Failed",
|
||||
413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
|
||||
416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
|
||||
421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
|
||||
424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
|
||||
428: "Precondition Required", 429: "Too Many Requests",
|
||||
431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
|
||||
500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
|
||||
503: "Service Unavailable", 504: "Gateway Timeout",
|
||||
505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
|
||||
507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
|
||||
511: "Network Authentication Required"
|
||||
100: "Continue",
|
||||
101: "Switching Protocols",
|
||||
102: "Processing",
|
||||
103: "Early Hints",
|
||||
200: "OK",
|
||||
201: "Created",
|
||||
202: "Accepted",
|
||||
203: "Non-Authoritative Information",
|
||||
204: "No Content",
|
||||
205: "Reset Content",
|
||||
206: "Partial Content",
|
||||
207: "Multi-Status",
|
||||
208: "Already Reported",
|
||||
226: "IM Used",
|
||||
300: "Multiple Choices",
|
||||
301: "Moved Permanently",
|
||||
302: "Found",
|
||||
303: "See Other",
|
||||
304: "Not Modified",
|
||||
305: "Use Proxy",
|
||||
307: "Temporary Redirect",
|
||||
308: "Permanent Redirect",
|
||||
400: "Bad Request",
|
||||
401: "Unauthorized",
|
||||
402: "Payment Required",
|
||||
403: "Forbidden",
|
||||
404: "Not Found",
|
||||
405: "Method Not Allowed",
|
||||
406: "Not Acceptable",
|
||||
407: "Proxy Authentication Required",
|
||||
408: "Request Timeout",
|
||||
409: "Conflict",
|
||||
410: "Gone",
|
||||
411: "Length Required",
|
||||
412: "Precondition Failed",
|
||||
413: "Payload Too Large",
|
||||
414: "URI Too Long",
|
||||
415: "Unsupported Media Type",
|
||||
416: "Range Not Satisfiable",
|
||||
417: "Expectation Failed",
|
||||
418: "I'm a teapot",
|
||||
421: "Misdirected Request",
|
||||
422: "Unprocessable Entity",
|
||||
423: "Locked",
|
||||
424: "Failed Dependency",
|
||||
425: "Too Early",
|
||||
426: "Upgrade Required",
|
||||
428: "Precondition Required",
|
||||
429: "Too Many Requests",
|
||||
431: "Request Header Fields Too Large",
|
||||
451: "Unavailable For Legal Reasons",
|
||||
500: "Internal Server Error",
|
||||
501: "Not Implemented",
|
||||
502: "Bad Gateway",
|
||||
503: "Service Unavailable",
|
||||
504: "Gateway Timeout",
|
||||
505: "HTTP Version Not Supported",
|
||||
506: "Variant Also Negotiates",
|
||||
507: "Insufficient Storage",
|
||||
508: "Loop Detected",
|
||||
510: "Not Extended",
|
||||
511: "Network Authentication Required",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from scrapling import Adaptor
|
||||
class TestParserAutoMatch:
|
||||
def test_element_relocation(self):
|
||||
"""Test relocating element after structure change"""
|
||||
original_html = '''
|
||||
original_html = """
|
||||
<div class="container">
|
||||
<section class="products">
|
||||
<article class="product" id="p1">
|
||||
@@ -21,8 +21,8 @@ class TestParserAutoMatch:
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
'''
|
||||
changed_html = '''
|
||||
"""
|
||||
changed_html = """
|
||||
<div class="new-container">
|
||||
<div class="product-wrapper">
|
||||
<section class="products">
|
||||
@@ -41,25 +41,25 @@ class TestParserAutoMatch:
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
'''
|
||||
"""
|
||||
|
||||
old_page = Adaptor(original_html, url='example.com', auto_match=True)
|
||||
new_page = Adaptor(changed_html, url='example.com', auto_match=True)
|
||||
old_page = Adaptor(original_html, url="example.com", auto_match=True)
|
||||
new_page = Adaptor(changed_html, url="example.com", auto_match=True)
|
||||
|
||||
# 'p1' was used as ID and now it's not and all the path elements have changes
|
||||
# Also at the same time testing auto-match vs combined selectors
|
||||
_ = old_page.css('#p1, #p2', auto_save=True)[0]
|
||||
relocated = new_page.css('#p1', auto_match=True)
|
||||
_ = old_page.css("#p1, #p2", auto_save=True)[0]
|
||||
relocated = new_page.css("#p1", auto_match=True)
|
||||
|
||||
assert relocated is not None
|
||||
assert relocated[0].attrib['data-id'] == 'p1'
|
||||
assert relocated[0].has_class('new-class')
|
||||
assert relocated[0].css('.new-description')[0].text == 'Description 1'
|
||||
assert relocated[0].attrib["data-id"] == "p1"
|
||||
assert relocated[0].has_class("new-class")
|
||||
assert relocated[0].css(".new-description")[0].text == "Description 1"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_element_relocation_async(self):
|
||||
"""Test relocating element after structure change in async mode"""
|
||||
original_html = '''
|
||||
original_html = """
|
||||
<div class="container">
|
||||
<section class="products">
|
||||
<article class="product" id="p1">
|
||||
@@ -72,8 +72,8 @@ class TestParserAutoMatch:
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
'''
|
||||
changed_html = '''
|
||||
"""
|
||||
changed_html = """
|
||||
<div class="new-container">
|
||||
<div class="product-wrapper">
|
||||
<section class="products">
|
||||
@@ -92,20 +92,20 @@ class TestParserAutoMatch:
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
'''
|
||||
"""
|
||||
|
||||
# Simulate async operation
|
||||
await asyncio.sleep(0.1) # Minimal async operation
|
||||
|
||||
old_page = Adaptor(original_html, url='example.com', auto_match=True)
|
||||
new_page = Adaptor(changed_html, url='example.com', auto_match=True)
|
||||
old_page = Adaptor(original_html, url="example.com", auto_match=True)
|
||||
new_page = Adaptor(changed_html, url="example.com", auto_match=True)
|
||||
|
||||
# 'p1' was used as ID and now it's not and all the path elements have changes
|
||||
# Also at the same time testing auto-match vs combined selectors
|
||||
_ = old_page.css('#p1, #p2', auto_save=True)[0]
|
||||
relocated = new_page.css('#p1', auto_match=True)
|
||||
_ = old_page.css("#p1, #p2", auto_save=True)[0]
|
||||
relocated = new_page.css("#p1", auto_match=True)
|
||||
|
||||
assert relocated is not None
|
||||
assert relocated[0].attrib['data-id'] == 'p1'
|
||||
assert relocated[0].has_class('new-class')
|
||||
assert relocated[0].css('.new-description')[0].text == 'Description 1'
|
||||
assert relocated[0].attrib["data-id"] == "p1"
|
||||
assert relocated[0].has_class("new-class")
|
||||
assert relocated[0].css(".new-description")[0].text == "Description 1"
|
||||
|
||||
@@ -9,7 +9,7 @@ from scrapling import Adaptor
|
||||
|
||||
@pytest.fixture
|
||||
def html_content():
|
||||
return '''
|
||||
return """
|
||||
<html>
|
||||
<head>
|
||||
<title>Complex Web Page</title>
|
||||
@@ -73,7 +73,7 @@ def html_content():
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -85,13 +85,14 @@ def page(html_content):
|
||||
class TestCSSSelectors:
|
||||
def test_basic_product_selection(self, page):
|
||||
"""Test selecting all product elements"""
|
||||
elements = page.css('main #products .product-list article.product')
|
||||
elements = page.css("main #products .product-list article.product")
|
||||
assert len(elements) == 3
|
||||
|
||||
def test_in_stock_product_selection(self, page):
|
||||
"""Test selecting in-stock products"""
|
||||
in_stock_products = page.css(
|
||||
'main #products .product-list article.product:not(:contains("Out of stock"))')
|
||||
'main #products .product-list article.product:not(:contains("Out of stock"))'
|
||||
)
|
||||
assert len(in_stock_products) == 2
|
||||
|
||||
|
||||
@@ -117,22 +118,26 @@ class TestXPathSelectors:
|
||||
class TestTextMatching:
|
||||
def test_regex_multiple_matches(self, page):
|
||||
"""Test finding multiple matches with regex"""
|
||||
stock_info = page.find_by_regex(r'In stock: \d+', first_match=False)
|
||||
stock_info = page.find_by_regex(r"In stock: \d+", first_match=False)
|
||||
assert len(stock_info) == 2
|
||||
|
||||
def test_regex_first_match(self, page):
|
||||
"""Test finding the first match with regex"""
|
||||
stock_info = page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
|
||||
assert stock_info.text == 'In stock: 5'
|
||||
stock_info = page.find_by_regex(
|
||||
r"In stock: \d+", first_match=True, case_sensitive=True
|
||||
)
|
||||
assert stock_info.text == "In stock: 5"
|
||||
|
||||
def test_partial_text_match(self, page):
|
||||
"""Test finding elements with partial text match"""
|
||||
stock_info = page.find_by_text(r'In stock:', partial=True, first_match=False)
|
||||
stock_info = page.find_by_text(r"In stock:", partial=True, first_match=False)
|
||||
assert len(stock_info) == 2
|
||||
|
||||
def test_exact_text_match(self, page):
|
||||
"""Test finding elements with exact text match"""
|
||||
out_of_stock = page.find_by_text('Out of stock', partial=False, first_match=False)
|
||||
out_of_stock = page.find_by_text(
|
||||
"Out of stock", partial=False, first_match=False
|
||||
)
|
||||
assert len(out_of_stock) == 1
|
||||
|
||||
|
||||
@@ -140,17 +145,17 @@ class TestTextMatching:
|
||||
class TestSimilarElements:
|
||||
def test_finding_similar_products(self, page):
|
||||
"""Test finding similar product elements"""
|
||||
first_product = page.css_first('.product')
|
||||
first_product = page.css_first(".product")
|
||||
similar_products = first_product.find_similar()
|
||||
assert len(similar_products) == 2
|
||||
|
||||
def test_finding_similar_reviews(self, page):
|
||||
"""Test finding similar review elements with additional filtering"""
|
||||
first_review = page.find('div', class_='review')
|
||||
first_review = page.find("div", class_="review")
|
||||
similar_high_rated_reviews = [
|
||||
review
|
||||
for review in first_review.find_similar()
|
||||
if int(review.attrib.get('data-rating', 0)) >= 4
|
||||
if int(review.attrib.get("data-rating", 0)) >= 4
|
||||
]
|
||||
assert len(similar_high_rated_reviews) == 1
|
||||
|
||||
@@ -181,17 +186,17 @@ class TestErrorHandling:
|
||||
def test_bad_selectors(self, page):
|
||||
"""Test handling of invalid selectors"""
|
||||
with pytest.raises((SelectorError, SelectorSyntaxError)):
|
||||
page.css('4 ayo')
|
||||
page.css("4 ayo")
|
||||
|
||||
with pytest.raises((SelectorError, SelectorSyntaxError)):
|
||||
page.xpath('4 ayo')
|
||||
page.xpath("4 ayo")
|
||||
|
||||
|
||||
# Pickling and Object Representation Tests
|
||||
class TestPicklingAndRepresentation:
|
||||
def test_unpickleable_objects(self, page):
|
||||
"""Test that Adaptor objects cannot be pickled"""
|
||||
table = page.css('.product-list')[0]
|
||||
table = page.css(".product-list")[0]
|
||||
with pytest.raises(TypeError):
|
||||
pickle.dumps(table)
|
||||
|
||||
@@ -200,7 +205,7 @@ class TestPicklingAndRepresentation:
|
||||
|
||||
def test_string_representations(self, page):
|
||||
"""Test custom string representations of objects"""
|
||||
table = page.css('.product-list')[0]
|
||||
table = page.css(".product-list")[0]
|
||||
assert issubclass(type(table.__str__()), str)
|
||||
assert issubclass(type(table.__repr__()), str)
|
||||
assert issubclass(type(table.attrib.__str__()), str)
|
||||
@@ -211,40 +216,40 @@ class TestPicklingAndRepresentation:
|
||||
class TestElementNavigation:
|
||||
def test_basic_navigation_properties(self, page):
|
||||
"""Test basic navigation properties of elements"""
|
||||
table = page.css('.product-list')[0]
|
||||
table = page.css(".product-list")[0]
|
||||
assert table.path is not None
|
||||
assert table.html_content != ''
|
||||
assert table.prettify() != ''
|
||||
assert table.html_content != ""
|
||||
assert table.prettify() != ""
|
||||
|
||||
def test_parent_and_sibling_navigation(self, page):
|
||||
"""Test parent and sibling navigation"""
|
||||
table = page.css('.product-list')[0]
|
||||
table = page.css(".product-list")[0]
|
||||
parent = table.parent
|
||||
assert parent.attrib['id'] == 'products'
|
||||
assert parent.attrib["id"] == "products"
|
||||
|
||||
parent_siblings = parent.siblings
|
||||
assert len(parent_siblings) == 1
|
||||
|
||||
def test_child_navigation(self, page):
|
||||
"""Test child navigation"""
|
||||
table = page.css('.product-list')[0]
|
||||
table = page.css(".product-list")[0]
|
||||
children = table.children
|
||||
assert len(children) == 3
|
||||
|
||||
def test_next_and_previous_navigation(self, page):
|
||||
"""Test next and previous element navigation"""
|
||||
child = page.css('.product-list')[0].find({'data-id': "1"})
|
||||
child = page.css(".product-list")[0].find({"data-id": "1"})
|
||||
next_element = child.next
|
||||
assert next_element.attrib['data-id'] == '2'
|
||||
assert next_element.attrib["data-id"] == "2"
|
||||
|
||||
prev_element = next_element.previous
|
||||
assert prev_element.tag == child.tag
|
||||
|
||||
def test_ancestor_finding(self, page):
|
||||
"""Test finding ancestors of elements"""
|
||||
all_prices = page.css('.price')
|
||||
all_prices = page.css(".price")
|
||||
products_with_prices = [
|
||||
price.find_ancestor(lambda p: p.has_class('product'))
|
||||
price.find_ancestor(lambda p: p.has_class("product"))
|
||||
for price in all_prices
|
||||
]
|
||||
assert len(products_with_prices) == 3
|
||||
@@ -254,52 +259,59 @@ class TestElementNavigation:
|
||||
class TestJSONAndAttributes:
|
||||
def test_json_conversion(self, page):
|
||||
"""Test converting content to JSON"""
|
||||
script_content = page.css('#page-data::text')[0]
|
||||
script_content = page.css("#page-data::text")[0]
|
||||
assert issubclass(type(script_content.sort()), str)
|
||||
page_data = script_content.json()
|
||||
assert page_data['totalProducts'] == 3
|
||||
assert 'lastUpdated' in page_data
|
||||
assert page_data["totalProducts"] == 3
|
||||
assert "lastUpdated" in page_data
|
||||
|
||||
def test_attribute_operations(self, page):
|
||||
"""Test various attribute-related operations"""
|
||||
# Product ID extraction
|
||||
products = page.css('.product')
|
||||
product_ids = [product.attrib['data-id'] for product in products]
|
||||
assert product_ids == ['1', '2', '3']
|
||||
assert 'data-id' in products[0].attrib
|
||||
products = page.css(".product")
|
||||
product_ids = [product.attrib["data-id"] for product in products]
|
||||
assert product_ids == ["1", "2", "3"]
|
||||
assert "data-id" in products[0].attrib
|
||||
|
||||
# Review rating calculations
|
||||
reviews = page.css('.review')
|
||||
review_ratings = [int(review.attrib['data-rating']) for review in reviews]
|
||||
reviews = page.css(".review")
|
||||
review_ratings = [int(review.attrib["data-rating"]) for review in reviews]
|
||||
assert sum(review_ratings) / len(review_ratings) == 4.5
|
||||
|
||||
# Attribute searching
|
||||
key_value = list(products[0].attrib.search_values('1', partial=False))
|
||||
assert list(key_value[0].keys()) == ['data-id']
|
||||
key_value = list(products[0].attrib.search_values("1", partial=False))
|
||||
assert list(key_value[0].keys()) == ["data-id"]
|
||||
|
||||
key_value = list(products[0].attrib.search_values('1', partial=True))
|
||||
assert list(key_value[0].keys()) == ['data-id']
|
||||
key_value = list(products[0].attrib.search_values("1", partial=True))
|
||||
assert list(key_value[0].keys()) == ["data-id"]
|
||||
|
||||
# JSON attribute conversion
|
||||
attr_json = page.css_first('#products').attrib['schema'].json()
|
||||
assert attr_json == {'jsonable': 'data'}
|
||||
assert isinstance(page.css('#products')[0].attrib.json_string, bytes)
|
||||
attr_json = page.css_first("#products").attrib["schema"].json()
|
||||
assert attr_json == {"jsonable": "data"}
|
||||
assert isinstance(page.css("#products")[0].attrib.json_string, bytes)
|
||||
|
||||
|
||||
# Performance Test
|
||||
def test_large_html_parsing_performance():
|
||||
"""Test parsing and selecting performance on large HTML"""
|
||||
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
|
||||
large_html = (
|
||||
"<html><body>"
|
||||
+ '<div class="item">' * 5000
|
||||
+ "</div>" * 5000
|
||||
+ "</body></html>"
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
parsed = Adaptor(large_html, auto_match=False)
|
||||
elements = parsed.css('.item')
|
||||
elements = parsed.css(".item")
|
||||
end_time = time.time()
|
||||
|
||||
assert len(elements) == 5000
|
||||
# Converting 5000 elements to a class and doing operations on them will take time
|
||||
# Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
|
||||
assert end_time - start_time < 0.5 # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
|
||||
assert (
|
||||
end_time - start_time < 0.5
|
||||
) # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
|
||||
|
||||
|
||||
# Selector Generation Test
|
||||
@@ -318,13 +330,13 @@ def test_selectors_generation(page):
|
||||
# Miscellaneous Tests
|
||||
def test_getting_all_text(page):
|
||||
"""Test getting all text from the page"""
|
||||
assert page.get_all_text() != ''
|
||||
assert page.get_all_text() != ""
|
||||
|
||||
|
||||
def test_regex_on_text(page):
|
||||
"""Test regex operations on text"""
|
||||
element = page.css('[data-id="1"] .price')[0]
|
||||
match = element.re_first(r'[\.\d]+')
|
||||
assert match == '10.99'
|
||||
match = element.text.re(r'(\d+)', replace_entities=False)
|
||||
match = element.re_first(r"[\.\d]+")
|
||||
assert match == "10.99"
|
||||
match = element.text.re(r"(\d+)", replace_entities=False)
|
||||
assert len(match) == 2
|
||||
|
||||
Reference in New Issue
Block a user