style: add flags for tests coverage

- Some are already tested but the coverage report can't see it.
- Some are not necessary to test or too hard to test on GitHub's CI
This commit is contained in:
Karim shoair
2025-08-17 01:02:14 +03:00
parent 009bbb20c0
commit 2f402f4835
13 changed files with 121 additions and 101 deletions
+3 -3
View File
@@ -15,7 +15,7 @@ __OUTPUT_FILE_HELP__ = "Output file path can be HTML content, Markdown of the HT
__PACKAGE_DIR__ = Path(__file__).parent __PACKAGE_DIR__ = Path(__file__).parent
def __Execute(cmd: List[str], help_line: str) -> None: def __Execute(cmd: List[str], help_line: str) -> None: # pragma: no cover
print(f"Installing {help_line}...") print(f"Installing {help_line}...")
_ = check_output(cmd, shell=False) # nosec B603 _ = check_output(cmd, shell=False) # nosec B603
# I meant to not use try except here # I meant to not use try except here
@@ -28,7 +28,7 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any
try: try:
return json_loads(json_string) return json_loads(json_string)
except JSONDecodeError as e: except JSONDecodeError as e: # pragma: no cover
raise ValueError(f"Invalid JSON data '{json_string}': {e}") raise ValueError(f"Invalid JSON data '{json_string}': {e}")
@@ -105,7 +105,7 @@ def __BuildRequest(
type=bool, type=bool,
help="Force Scrapling to reinstall all Fetchers dependencies", help="Force Scrapling to reinstall all Fetchers dependencies",
) )
def install(force): def install(force): # pragma: no cover
if ( if (
force force
or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists() or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists()
+1 -1
View File
@@ -340,7 +340,7 @@ def _replace_entities(
if 0x80 <= number <= 0x9F: if 0x80 <= number <= 0x9F:
return bytes((number,)).decode("cp1252") return bytes((number,)).decode("cp1252")
return chr(number) return chr(number)
except (ValueError, OverflowError): except (ValueError, OverflowError): # pragma: no cover
pass pass
return "" if remove_illegal and groups.get("semicolon") else m.group(0) return "" if remove_illegal and groups.get("semicolon") else m.group(0)
+1 -1
View File
@@ -35,7 +35,7 @@ StrOrBytes = Union[str, bytes]
try: try:
# Python 3.11+ # Python 3.11+
from typing import Self # novermin from typing import Self # novermin
except ImportError: except ImportError: # pragma: no cover
try: try:
from typing_extensions import Self # Backport from typing_extensions import Self # Backport
except ImportError: except ImportError:
+39 -23
View File
@@ -31,11 +31,15 @@ class TextHandler(str):
__slots__ = () __slots__ = ()
def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": def __getitem__(
self, key: SupportsIndex | slice
) -> "TextHandler": # pragma: no cover
lst = super().__getitem__(key) lst = super().__getitem__(key)
return cast(_TextHandlerType, TextHandler(lst)) return cast(_TextHandlerType, TextHandler(lst))
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": def split(
self, sep: str = None, maxsplit: SupportsIndex = -1
) -> "TextHandlers": # pragma: no cover
return TextHandlers( return TextHandlers(
cast( cast(
List[_TextHandlerType], List[_TextHandlerType],
@@ -43,58 +47,70 @@ class TextHandler(str):
) )
) )
def strip(self, chars: str = None) -> Union[str, "TextHandler"]: def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().strip(chars)) return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: def lstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().lstrip(chars)) return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: def rstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rstrip(chars)) return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, "TextHandler"]: def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().capitalize()) return TextHandler(super().capitalize())
def casefold(self) -> Union[str, "TextHandler"]: def casefold(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().casefold()) return TextHandler(super().casefold())
def center( def center(
self, width: SupportsIndex, fillchar: str = " " self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: ) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().center(width, fillchar)) return TextHandler(super().center(width, fillchar))
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: def expandtabs(
self, tabsize: SupportsIndex = 8
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().expandtabs(tabsize)) return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: def format(
self, *args: str, **kwargs: str
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format(*args, **kwargs)) return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, "TextHandler"]: def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format_map(mapping)) return TextHandler(super().format_map(mapping))
def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]: def join(
self, iterable: Iterable[str]
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().join(iterable)) return TextHandler(super().join(iterable))
def ljust( def ljust(
self, width: SupportsIndex, fillchar: str = " " self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: ) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().ljust(width, fillchar)) return TextHandler(super().ljust(width, fillchar))
def rjust( def rjust(
self, width: SupportsIndex, fillchar: str = " " self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: ) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rjust(width, fillchar)) return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, "TextHandler"]: def swapcase(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().swapcase()) return TextHandler(super().swapcase())
def title(self) -> Union[str, "TextHandler"]: def title(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().title()) return TextHandler(super().title())
def translate(self, table) -> Union[str, "TextHandler"]: def translate(self, table) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().translate(table)) return TextHandler(super().translate(table))
def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]: def zfill(
self, width: SupportsIndex
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().zfill(width)) return TextHandler(super().zfill(width))
def replace( def replace(
@@ -120,10 +136,10 @@ class TextHandler(str):
return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip()) return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
# For easy copy-paste from Scrapy/parsel code when needed :) # For easy copy-paste from Scrapy/parsel code when needed :)
def get(self, default=None): def get(self, default=None): # pragma: no cover
return self return self
def get_all(self): def get_all(self): # pragma: no cover
return self return self
extract = get_all extract = get_all
@@ -234,11 +250,11 @@ class TextHandlers(List[TextHandler]):
__slots__ = () __slots__ = ()
@overload @overload
def __getitem__(self, pos: SupportsIndex) -> TextHandler: def __getitem__(self, pos: SupportsIndex) -> TextHandler: # pragma: no cover
pass pass
@overload @overload
def __getitem__(self, pos: slice) -> "TextHandlers": def __getitem__(self, pos: slice) -> "TextHandlers": # pragma: no cover
pass pass
def __getitem__( def __getitem__(
@@ -276,7 +292,7 @@ class TextHandlers(List[TextHandler]):
replace_entities: bool = True, replace_entities: bool = True,
clean_match: bool = False, clean_match: bool = False,
case_sensitive: bool = True, case_sensitive: bool = True,
) -> TextHandler: ) -> TextHandler: # pragma: no cover
"""Call the ``.re_first()`` method for each element in this list and return """Call the ``.re_first()`` method for each element in this list and return
the first result or the default value otherwise. the first result or the default value otherwise.
+20 -22
View File
@@ -108,7 +108,7 @@ def _ParseHeaders(
cookie_dict = { cookie_dict = {
key: value for key, value in _CookieParser(header_value) key: value for key, value in _CookieParser(header_value)
} }
except Exception as e: except Exception as e: # pragma: no cover
raise ValueError( raise ValueError(
f"Could not parse cookie string from header '{header_value}': {e}" f"Could not parse cookie string from header '{header_value}': {e}"
) )
@@ -121,7 +121,7 @@ def _ParseHeaders(
# Suppress exit on error to handle parsing errors gracefully # Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser): class NoExitArgumentParser(ArgumentParser): # pragma: no cover
def error(self, message): def error(self, message):
log.error(f"Curl arguments parsing error: {message}") log.error(f"Curl arguments parsing error: {message}")
raise ValueError(f"Curl arguments parsing error: {message}") raise ValueError(f"Curl arguments parsing error: {message}")
@@ -188,8 +188,6 @@ class CurlParser:
self.parser: NoExitArgumentParser = _parser self.parser: NoExitArgumentParser = _parser
self._supported_methods = ("get", "post", "put", "delete") self._supported_methods = ("get", "post", "put", "delete")
# --- Helper Functions ---
# --- Main Parsing Logic --- # --- Main Parsing Logic ---
def parse(self, curl_command: str) -> Optional[Request]: def parse(self, curl_command: str) -> Optional[Request]:
"""Parses the curl command string into a structured context for Fetcher.""" """Parses the curl command string into a structured context for Fetcher."""
@@ -200,7 +198,7 @@ class CurlParser:
tokens = shlex_split( tokens = shlex_split(
clean_command clean_command
) # Split the string using shell-like syntax ) # Split the string using shell-like syntax
except ValueError as e: except ValueError as e: # pragma: no cover
log.error(f"Could not split command line: {e}") log.error(f"Could not split command line: {e}")
return None return None
@@ -209,13 +207,13 @@ class CurlParser:
if unknown: if unknown:
raise AttributeError(f"Unknown/Unsupported curl arguments: {unknown}") raise AttributeError(f"Unknown/Unsupported curl arguments: {unknown}")
except ValueError: except ValueError: # pragma: no cover
return None return None
except AttributeError: except AttributeError:
raise raise
except Exception as e: except Exception as e: # pragma: no cover
log.error( log.error(
f"An unexpected error occurred during curl arguments parsing: {e}" f"An unexpected error occurred during curl arguments parsing: {e}"
) )
@@ -249,7 +247,7 @@ class CurlParser:
# Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:' # Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:'
cookies[key] = value cookies[key] = value
log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}") log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
except Exception as e: except Exception as e: # pragma: no cover
log.error( log.error(
f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}" f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}"
) )
@@ -261,7 +259,7 @@ class CurlParser:
# DevTools often uses --data-raw for JSON bodies # DevTools often uses --data-raw for JSON bodies
# Precedence: --data-binary > --data-raw / -d > --data-urlencode # Precedence: --data-binary > --data-raw / -d > --data-urlencode
if parsed_args.data_binary is not None: if parsed_args.data_binary is not None: # pragma: no cover
try: try:
data_payload = parsed_args.data_binary.encode("utf-8") data_payload = parsed_args.data_binary.encode("utf-8")
log.debug("Using data from --data-binary as bytes.") log.debug("Using data from --data-binary as bytes.")
@@ -277,7 +275,7 @@ class CurlParser:
elif parsed_args.data is not None: elif parsed_args.data is not None:
data_payload = parsed_args.data data_payload = parsed_args.data
elif parsed_args.data_urlencode: elif parsed_args.data_urlencode: # pragma: no cover
# Combine and parse urlencoded data # Combine and parse urlencoded data
combined_data = "&".join(parsed_args.data_urlencode) combined_data = "&".join(parsed_args.data_urlencode)
try: try:
@@ -299,7 +297,7 @@ class CurlParser:
pass # Not JSON, keep it in data_payload pass # Not JSON, keep it in data_payload
# Handle `-G`: Move data to params if the method is GET # Handle `-G`: Move data to params if the method is GET
if method == "get" and data_payload: if method == "get" and data_payload: # pragma: no cover
if isinstance(data_payload, dict): # From --data-urlencode likely if isinstance(data_payload, dict): # From --data-urlencode likely
params.update(data_payload) params.update(data_payload)
elif isinstance(data_payload, str): elif isinstance(data_payload, str):
@@ -369,7 +367,7 @@ class CurlParser:
) )
# Ensure request parsing was successful before proceeding # Ensure request parsing was successful before proceeding
if request is None: if request is None: # pragma: no cover
log.error("Failed to parse curl command, cannot convert to fetcher.") log.error("Failed to parse curl command, cannot convert to fetcher.")
return None return None
@@ -385,22 +383,22 @@ class CurlParser:
try: try:
return getattr(Fetcher, method)(**request_args) return getattr(Fetcher, method)(**request_args)
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error calling Fetcher.{method}: {e}") log.error(f"Error calling Fetcher.{method}: {e}")
return None return None
else: else: # pragma: no cover
log.error( log.error(
f'Request method "{method}" isn\'t supported by Scrapling yet' f'Request method "{method}" isn\'t supported by Scrapling yet'
) )
return None return None
else: else: # pragma: no cover
log.error("Input must be a valid curl command string or a Request object.") log.error("Input must be a valid curl command string or a Request object.")
return None return None
def show_page_in_browser(page: Selector): def show_page_in_browser(page: Selector): # pragma: no cover
if not page or not isinstance(page, Selector): if not page or not isinstance(page, Selector):
log.error("Input must be of type `Selector`") log.error("Input must be of type `Selector`")
return return
@@ -429,7 +427,7 @@ class CustomShell:
if _known_logging_levels.get(log_level): if _known_logging_levels.get(log_level):
self.log_level = _known_logging_levels[log_level] self.log_level = _known_logging_levels[log_level]
else: else: # pragma: no cover
log.warning(f'Unknown log level "{log_level}", defaulting to "DEBUG"') log.warning(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
self.log_level = DEBUG self.log_level = DEBUG
@@ -480,7 +478,7 @@ class CustomShell:
Type 'exit' or press Ctrl+D to exit. Type 'exit' or press Ctrl+D to exit.
""" """
def update_page(self, result): def update_page(self, result): # pragma: no cover
"""Update the current page and add to pages history""" """Update the current page and add to pages history"""
self.page = result self.page = result
if isinstance(result, (Response, Selector)): if isinstance(result, (Response, Selector)):
@@ -540,11 +538,11 @@ Type 'exit' or press Ctrl+D to exit.
"help": self.show_help, "help": self.show_help,
} }
def show_help(self): def show_help(self): # pragma: no cover
"""Show help information""" """Show help information"""
print(self.banner()) print(self.banner())
def start(self): def start(self): # pragma: no cover
"""Start the interactive shell""" """Start the interactive shell"""
# Get our namespace with application objects # Get our namespace with application objects
namespace = self.get_namespace() namespace = self.get_namespace()
@@ -594,7 +592,7 @@ class Convertor:
main_content_only: bool = False, main_content_only: bool = False,
) -> Generator[str, None, None]: ) -> Generator[str, None, None]:
"""Extract the content of a Selector""" """Extract the content of a Selector"""
if not page or not isinstance(page, Selector): if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`") raise TypeError("Input must be of type `Selector`")
elif not extraction_type or extraction_type not in cls._extension_map.values(): elif not extraction_type or extraction_type not in cls._extension_map.values():
raise ValueError(f"Unknown extraction type: {extraction_type}") raise ValueError(f"Unknown extraction type: {extraction_type}")
@@ -627,7 +625,7 @@ class Convertor:
cls, page: Selector, filename: str, css_selector: Optional[str] = None cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None: ) -> None:
"""Write a Selector's content to a file""" """Write a Selector's content to a file"""
if not page or not isinstance(page, Selector): if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`") raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip(): elif not filename or not isinstance(filename, str) or not filename.strip():
raise ValueError("Filename must be provided") raise ValueError("Filename must be provided")
+1 -1
View File
@@ -12,7 +12,7 @@ from scrapling.core.utils import _StorageTools, log
from scrapling.core._types import Dict, Optional, Any from scrapling.core._types import Dict, Optional, Any
class StorageSystemMixin(ABC): class StorageSystemMixin(ABC): # pragma: no cover
# If you want to make your own storage system, you have to inherit from this # If you want to make your own storage system, you have to inherit from this
def __init__(self, url: Optional[str] = None): def __init__(self, url: Optional[str] = None):
""" """
+9 -9
View File
@@ -37,15 +37,15 @@ class XPathExpr(OriginalXPathExpr):
def __str__(self) -> str: def __str__(self) -> str:
path = super().__str__() path = super().__str__()
if self.textnode: if self.textnode:
if path == "*": if path == "*": # pragma: no cover
path = "text()" path = "text()"
elif path.endswith("::*/*"): elif path.endswith("::*/*"): # pragma: no cover
path = path[:-3] + "text()" path = path[:-3] + "text()"
else: else:
path += "/text()" path += "/text()"
if self.attribute is not None: if self.attribute is not None:
if path.endswith("::*/*"): if path.endswith("::*/*"): # pragma: no cover
path = path[:-2] path = path[:-2]
path += f"/@{self.attribute}" path += f"/@{self.attribute}"
@@ -59,7 +59,7 @@ class XPathExpr(OriginalXPathExpr):
**kwargs: Any, **kwargs: Any,
) -> Self: ) -> Self:
if not isinstance(other, XPathExpr): if not isinstance(other, XPathExpr):
raise ValueError( raise ValueError( # pragma: no cover
f"Expressions of type {__name__}.XPathExpr can ony join expressions" f"Expressions of type {__name__}.XPathExpr can ony join expressions"
f" of the same type (or its descendants), got {type(other)}" f" of the same type (or its descendants), got {type(other)}"
) )
@@ -71,10 +71,10 @@ class XPathExpr(OriginalXPathExpr):
# e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator # e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator
class TranslatorProtocol(Protocol): class TranslatorProtocol(Protocol):
def xpath_element(self, selector: Element) -> OriginalXPathExpr: def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pragma: no cover
pass pass
def css_to_xpath(self, css: str, prefix: str = ...) -> str: def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pragma: no cover
pass pass
@@ -98,7 +98,7 @@ class TranslatorMixin:
if isinstance(pseudo_element, FunctionalPseudoElement): if isinstance(pseudo_element, FunctionalPseudoElement):
method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element" method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
method = getattr(self, method_name, None) method = getattr(self, method_name, None)
if not method: if not method: # pragma: no cover
raise ExpressionError( raise ExpressionError(
f"The functional pseudo-element ::{pseudo_element.name}() is unknown" f"The functional pseudo-element ::{pseudo_element.name}() is unknown"
) )
@@ -108,7 +108,7 @@ class TranslatorMixin:
f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element" f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
) )
method = getattr(self, method_name, None) method = getattr(self, method_name, None)
if not method: if not method: # pragma: no cover
raise ExpressionError( raise ExpressionError(
f"The pseudo-element ::{pseudo_element} is unknown" f"The pseudo-element ::{pseudo_element} is unknown"
) )
@@ -120,7 +120,7 @@ class TranslatorMixin:
xpath: OriginalXPathExpr, function: FunctionalPseudoElement xpath: OriginalXPathExpr, function: FunctionalPseudoElement
) -> XPathExpr: ) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element""" """Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]): if function.argument_types() not in (["STRING"], ["IDENT"]): # pragma: no cover
raise ExpressionError( raise ExpressionError(
f"Expected a single string or ident for ::attr(), got {function.arguments!r}" f"Expected a single string or ident for ::attr(), got {function.arguments!r}"
) )
+17 -15
View File
@@ -229,22 +229,24 @@ class StealthySession:
def __create__(self): def __create__(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
self.playwright = sync_playwright().start() self.playwright = sync_playwright().start()
self.context = self.playwright.firefox.launch_persistent_context( self.context = (
**self.launch_options self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
) )
if self.cookies: if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies) self.context.add_cookies(self.cookies)
def __enter__(self): def __enter__(self): # pragma: no cover
self.__create__() self.__create__()
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
self.close() self.close()
def close(self): def close(self): # pragma: no cover
"""Close all resources""" """Close all resources"""
if self._closed: if self._closed: # pragma: no cover
return return
if self.context: if self.context:
@@ -257,7 +259,7 @@ class StealthySession:
self._closed = True self._closed = True
def _get_or_create_page(self) -> PageInfo: def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one""" """Get an available page or create a new one"""
# Try to get a ready page first # Try to get a ready page first
page_info = self.page_pool.get_ready_page() page_info = self.page_pool.get_ready_page()
@@ -319,7 +321,7 @@ class StealthySession:
return None return None
def _solve_cloudflare(self, page: Page) -> None: def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed """Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page :param page: The targeted page
@@ -371,7 +373,7 @@ class StealthySession:
:param url: The Target url. :param url: The Target url.
:return: A `Response` object. :return: A `Response` object.
""" """
if self._closed: if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed") raise RuntimeError("Context manager has been closed")
final_response = None final_response = None
@@ -392,7 +394,7 @@ class StealthySession:
page_info = self._get_or_create_page() page_info = self._get_or_create_page()
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: try: # pragma: no cover
# Navigate to URL and wait for a specified state # Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response) page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer) first_response = page_info.page.goto(url, referer=referer)
@@ -440,7 +442,7 @@ class StealthySession:
return response return response
except Exception as e: except Exception as e: # pragma: no cover
page_info.mark_error() page_info.mark_error()
raise e raise e
@@ -567,7 +569,7 @@ class AsyncStealthySession(StealthySession):
async def close(self): async def close(self):
"""Close all resources""" """Close all resources"""
if self._closed: if self._closed: # pragma: no cover
return return
if self.context: if self.context:
@@ -605,7 +607,7 @@ class AsyncStealthySession(StealthySession):
max_wait = 30 max_wait = 30
start_time = time() start_time = time()
while time() - start_time < max_wait: while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page() page_info = self.page_pool.get_ready_page()
if page_info: if page_info:
return page_info return page_info
@@ -625,7 +627,7 @@ class AsyncStealthySession(StealthySession):
return return
else: else:
log.info(f'The turnstile version discovered is "{challenge_type}"') log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive": if challenge_type == "non-interactive": # pragma: no cover
while "<title>Just a moment...</title>" in (await page.content()): while "<title>Just a moment...</title>" in (await page.content()):
log.info("Waiting for Cloudflare wait page to disappear.") log.info("Waiting for Cloudflare wait page to disappear.")
await page.wait_for_timeout(1000) await page.wait_for_timeout(1000)
@@ -667,7 +669,7 @@ class AsyncStealthySession(StealthySession):
:param url: The Target url. :param url: The Target url.
:return: A `Response` object. :return: A `Response` object.
""" """
if self._closed: if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed") raise RuntimeError("Context manager has been closed")
final_response = None final_response = None
+1 -1
View File
@@ -40,7 +40,7 @@ def _compiled_stealth_scripts():
@lru_cache(2, typed=True) @lru_cache(2, typed=True)
def _set_flags(hide_canvas, disable_webgl): def _set_flags(hide_canvas, disable_webgl): # pragma: no cover
"""Returns the flags that will be used while launching the browser if stealth mode is enabled""" """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS flags = DEFAULT_STEALTH_FLAGS
if hide_canvas: if hide_canvas:
+12 -12
View File
@@ -234,7 +234,7 @@ class DynamicSession:
self.playwright = sync_context().start() self.playwright = sync_context().start()
if self.cdp_url: if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp( self.context = self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url endpoint_url=self.cdp_url
).new_context(**self.context_options) ).new_context(**self.context_options)
@@ -243,7 +243,7 @@ class DynamicSession:
user_data_dir="", **self.launch_options user_data_dir="", **self.launch_options
) )
if self.cookies: if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies) self.context.add_cookies(self.cookies)
def __enter__(self): def __enter__(self):
@@ -253,7 +253,7 @@ class DynamicSession:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
self.close() self.close()
def close(self): def close(self): # pragma: no cover
"""Close all resources""" """Close all resources"""
if self._closed: if self._closed:
return return
@@ -268,7 +268,7 @@ class DynamicSession:
self._closed = True self._closed = True
def _get_or_create_page(self) -> PageInfo: def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one""" """Get an available page or create a new one"""
# Try to get a ready page first # Try to get a ready page first
page_info = self.page_pool.get_ready_page() page_info = self.page_pool.get_ready_page()
@@ -310,7 +310,7 @@ class DynamicSession:
:param url: The Target url. :param url: The Target url.
:return: A `Response` object. :return: A `Response` object.
""" """
if self._closed: if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed") raise RuntimeError("Context manager has been closed")
final_response = None final_response = None
@@ -331,7 +331,7 @@ class DynamicSession:
page_info = self._get_or_create_page() page_info = self._get_or_create_page()
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: try: # pragma: no cover
# Navigate to URL and wait for a specified state # Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response) page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer) first_response = page_info.page.goto(url, referer=referer)
@@ -346,7 +346,7 @@ class DynamicSession:
if self.page_action is not None: if self.page_action is not None:
try: try:
page_info.page = self.page_action(page_info.page) page_info.page = self.page_action(page_info.page)
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if self.wait_selector: if self.wait_selector:
@@ -358,7 +358,7 @@ class DynamicSession:
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page_info.page.wait_for_timeout(self.wait) page_info.page.wait_for_timeout(self.wait)
@@ -506,7 +506,7 @@ class AsyncDynamicSession(DynamicSession):
async def close(self): async def close(self):
"""Close all resources""" """Close all resources"""
if self._closed: if self._closed: # pragma: no cover
return return
if self.context: if self.context:
@@ -548,7 +548,7 @@ class AsyncDynamicSession(DynamicSession):
max_wait = 30 # seconds max_wait = 30 # seconds
start_time = time() start_time = time()
while time() - start_time < max_wait: while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page() page_info = self.page_pool.get_ready_page()
if page_info: if page_info:
return page_info return page_info
@@ -562,7 +562,7 @@ class AsyncDynamicSession(DynamicSession):
:param url: The Target url. :param url: The Target url.
:return: A `Response` object. :return: A `Response` object.
""" """
if self._closed: if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed") raise RuntimeError("Context manager has been closed")
final_response = None final_response = None
@@ -625,6 +625,6 @@ class AsyncDynamicSession(DynamicSession):
return response return response
except Exception as e: except Exception as e: # pragma: no cover
page_info.mark_error() page_info.mark_error()
raise e raise e
+7 -5
View File
@@ -112,7 +112,9 @@ class FetcherSession:
kwargs, "impersonate", self.default_impersonate kwargs, "impersonate", self.default_impersonate
) )
if self.get_with_precedence(kwargs, "http3", self.default_http3): if self.get_with_precedence(
kwargs, "http3", self.default_http3
): # pragma: no cover
request_args["http_version"] = CurlHttpVersion.V3ONLY request_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate: if impersonate:
log.warning( log.warning(
@@ -286,7 +288,7 @@ class FetcherSession:
response = session.request(method, **request_args) response = session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400 # response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config) return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: except CurlError as e: # pragma: no cover
if attempt < max_retries - 1: if attempt < max_retries - 1:
log.error( log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
@@ -296,7 +298,7 @@ class FetcherSession:
log.error(f"Failed after {max_retries} attempts: {e}") log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.") raise RuntimeError("No active session available.") # pragma: no cover
async def __make_async_request( async def __make_async_request(
self, self,
@@ -333,7 +335,7 @@ class FetcherSession:
response = await session.request(method, **request_args) response = await session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400 # response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config) return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: except CurlError as e: # pragma: no cover
if attempt < max_retries - 1: if attempt < max_retries - 1:
log.error( log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
@@ -343,7 +345,7 @@ class FetcherSession:
log.error(f"Failed after {max_retries} attempts: {e}") log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.") raise RuntimeError("No active session available.") # pragma: no cover
@staticmethod @staticmethod
def get_with_precedence(kwargs, key, default_value): def get_with_precedence(kwargs, key, default_value):
+6 -6
View File
@@ -52,12 +52,12 @@ class ResponseFactory:
**parser_arguments, **parser_arguments,
), ),
) )
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
current_request = current_request.redirected_from current_request = current_request.redirected_from
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}") log.error(f"Error processing response history: {e}")
return history return history
@@ -105,7 +105,7 @@ class ResponseFactory:
history = cls._process_response_history(first_response, parser_arguments) history = cls._process_response_history(first_response, parser_arguments)
try: try:
page_content = page.content() page_content = page.content()
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}") log.error(f"Error getting page content: {e}")
page_content = "" page_content = ""
@@ -157,12 +157,12 @@ class ResponseFactory:
**parser_arguments, **parser_arguments,
), ),
) )
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
current_request = current_request.redirected_from current_request = current_request.redirected_from
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}") log.error(f"Error processing response history: {e}")
return history return history
@@ -212,7 +212,7 @@ class ResponseFactory:
) )
try: try:
page_content = await page.content() page_content = await page.content()
except Exception as e: except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}") log.error(f"Error getting page content in async: {e}")
page_content = "" page_content = ""
+4 -2
View File
@@ -169,7 +169,9 @@ class Selector(SelectorsGeneration):
"Storage class must be wrapped with lru_cache decorator, see docs for info" "Storage class must be wrapped with lru_cache decorator, see docs for info"
) )
if not issubclass(storage.__wrapped__, StorageSystemMixin): if not issubclass(
storage.__wrapped__, StorageSystemMixin
): # pragma: no cover
raise ValueError( raise ValueError(
"Storage system must be inherited from class `StorageSystemMixin`" "Storage system must be inherited from class `StorageSystemMixin`"
) )
@@ -1400,7 +1402,7 @@ class Selectors(List[Selector]):
"""Returns the length of the current list""" """Returns the length of the current list"""
return len(self) return len(self)
def __getstate__(self) -> Any: def __getstate__(self) -> Any: # pragma: no cover
# lxml don't like it :) # lxml don't like it :)
raise TypeError("Can't pickle Selectors object") raise TypeError("Can't pickle Selectors object")