freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: add flags for tests coverage

Browse Source 
- Some are already tested but the coverage report can't see it.
- Some are not necessary to test or too hard to test on GitHub's CI
This commit is contained in:
Karim shoair
2025-08-17 01:02:14 +03:00
parent 009bbb20c0
commit 2f402f4835
13 changed files with 121 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+17 -15
View File
@@ -229,22 +229,24 @@ class StealthySession:
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
self.context = self.playwright.firefox.launch_persistent_context(
**self.launch_options
self.context = (
self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
)
if self.cookies:
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
def __enter__(self):
def __enter__(self): # pragma: no cover
self.__create__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
def close(self): # pragma: no cover
"""Close all resources"""
if self._closed:
if self._closed: # pragma: no cover
return
if self.context:
@@ -257,7 +259,7 @@ class StealthySession:
self._closed = True
def _get_or_create_page(self) -> PageInfo:
def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one"""
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
@@ -319,7 +321,7 @@ class StealthySession:
return None
def _solve_cloudflare(self, page: Page) -> None:
def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page
@@ -371,7 +373,7 @@ class StealthySession:
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed:
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
@@ -392,7 +394,7 @@ class StealthySession:
page_info = self._get_or_create_page()
page_info.mark_busy(url=url)
try:
try: # pragma: no cover
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer)
@@ -440,7 +442,7 @@ class StealthySession:
return response
except Exception as e:
except Exception as e: # pragma: no cover
page_info.mark_error()
raise e
@@ -567,7 +569,7 @@ class AsyncStealthySession(StealthySession):
async def close(self):
"""Close all resources"""
if self._closed:
if self._closed: # pragma: no cover
return
if self.context:
@@ -605,7 +607,7 @@ class AsyncStealthySession(StealthySession):
max_wait = 30
start_time = time()
while time() - start_time < max_wait:
while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
@@ -625,7 +627,7 @@ class AsyncStealthySession(StealthySession):
return
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
if challenge_type == "non-interactive": # pragma: no cover
while "<title>Just a moment...</title>" in (await page.content()):
log.info("Waiting for Cloudflare wait page to disappear.")
await page.wait_for_timeout(1000)
@@ -667,7 +669,7 @@ class AsyncStealthySession(StealthySession):
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed:
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
scrapling/engines/_browsers/_config_tools.py
+1 -1
View File
@@ -40,7 +40,7 @@ def _compiled_stealth_scripts():
@lru_cache(2, typed=True)
def _set_flags(hide_canvas, disable_webgl):
def _set_flags(hide_canvas, disable_webgl): # pragma: no cover
"""Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS
if hide_canvas:
scrapling/engines/_browsers/_controllers.py
+12 -12
View File
@@ -234,7 +234,7 @@ class DynamicSession:
self.playwright = sync_context().start()
if self.cdp_url:
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url
).new_context(**self.context_options)
@@ -243,7 +243,7 @@ class DynamicSession:
user_data_dir="", **self.launch_options
)
if self.cookies:
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
def __enter__(self):
@@ -253,7 +253,7 @@ class DynamicSession:
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
def close(self): # pragma: no cover
"""Close all resources"""
if self._closed:
return
@@ -268,7 +268,7 @@ class DynamicSession:
self._closed = True
def _get_or_create_page(self) -> PageInfo:
def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one"""
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
@@ -310,7 +310,7 @@ class DynamicSession:
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed:
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
@@ -331,7 +331,7 @@ class DynamicSession:
page_info = self._get_or_create_page()
page_info.mark_busy(url=url)
try:
try: # pragma: no cover
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer)
@@ -346,7 +346,7 @@ class DynamicSession:
if self.page_action is not None:
try:
page_info.page = self.page_action(page_info.page)
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
if self.wait_selector:
@@ -358,7 +358,7 @@ class DynamicSession:
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page_info.page.wait_for_timeout(self.wait)
@@ -506,7 +506,7 @@ class AsyncDynamicSession(DynamicSession):
async def close(self):
"""Close all resources"""
if self._closed:
if self._closed: # pragma: no cover
return
if self.context:
@@ -548,7 +548,7 @@ class AsyncDynamicSession(DynamicSession):
max_wait = 30 # seconds
start_time = time()
while time() - start_time < max_wait:
while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
@@ -562,7 +562,7 @@ class AsyncDynamicSession(DynamicSession):
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed:
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
@@ -625,6 +625,6 @@ class AsyncDynamicSession(DynamicSession):
return response
except Exception as e:
except Exception as e: # pragma: no cover
page_info.mark_error()
raise e
scrapling/engines/static.py
+7 -5
View File
@@ -112,7 +112,9 @@ class FetcherSession:
kwargs, "impersonate", self.default_impersonate
)
if self.get_with_precedence(kwargs, "http3", self.default_http3):
if self.get_with_precedence(
kwargs, "http3", self.default_http3
): # pragma: no cover
request_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate:
log.warning(
@@ -286,7 +288,7 @@ class FetcherSession:
response = session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e:
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
@@ -296,7 +298,7 @@ class FetcherSession:
log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.")
raise RuntimeError("No active session available.") # pragma: no cover
async def __make_async_request(
self,
@@ -333,7 +335,7 @@ class FetcherSession:
response = await session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e:
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
@@ -343,7 +345,7 @@ class FetcherSession:
log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.")
raise RuntimeError("No active session available.") # pragma: no cover
@staticmethod
def get_with_precedence(kwargs, key, default_value):
scrapling/engines/toolbelt/convertor.py
+6 -6
View File
@@ -52,12 +52,12 @@ class ResponseFactory:
**parser_arguments,
),
)
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}")
return history
@@ -105,7 +105,7 @@ class ResponseFactory:
history = cls._process_response_history(first_response, parser_arguments)
try:
page_content = page.content()
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}")
page_content = ""
@@ -157,12 +157,12 @@ class ResponseFactory:
**parser_arguments,
),
)
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}")
return history
@@ -212,7 +212,7 @@ class ResponseFactory:
)
try:
page_content = await page.content()
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}")
page_content = ""