freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: add flags for tests coverage

Browse Source 
- Some are already tested but the coverage report can't see it.
- Some are not necessary to test or too hard to test on GitHub's CI
This commit is contained in:
Karim shoair
2025-08-17 01:02:14 +03:00
parent 009bbb20c0
commit 2f402f4835
13 changed files with 121 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/shell.py
+20 -22
View File
@@ -108,7 +108,7 @@ def _ParseHeaders(
cookie_dict = {
key: value for key, value in _CookieParser(header_value)
}
except Exception as e:
except Exception as e: # pragma: no cover
raise ValueError(
f"Could not parse cookie string from header '{header_value}': {e}"
)
@@ -121,7 +121,7 @@ def _ParseHeaders(
# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser):
class NoExitArgumentParser(ArgumentParser): # pragma: no cover
def error(self, message):
log.error(f"Curl arguments parsing error: {message}")
raise ValueError(f"Curl arguments parsing error: {message}")
@@ -188,8 +188,6 @@ class CurlParser:
self.parser: NoExitArgumentParser = _parser
self._supported_methods = ("get", "post", "put", "delete")
# --- Helper Functions ---
# --- Main Parsing Logic ---
def parse(self, curl_command: str) -> Optional[Request]:
"""Parses the curl command string into a structured context for Fetcher."""
@@ -200,7 +198,7 @@ class CurlParser:
tokens = shlex_split(
clean_command
) # Split the string using shell-like syntax
except ValueError as e:
except ValueError as e: # pragma: no cover
log.error(f"Could not split command line: {e}")
return None
@@ -209,13 +207,13 @@ class CurlParser:
if unknown:
raise AttributeError(f"Unknown/Unsupported curl arguments: {unknown}")
except ValueError:
except ValueError: # pragma: no cover
return None
except AttributeError:
raise
except Exception as e:
except Exception as e: # pragma: no cover
log.error(
f"An unexpected error occurred during curl arguments parsing: {e}"
)
@@ -249,7 +247,7 @@ class CurlParser:
# Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:'
cookies[key] = value
log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
except Exception as e:
except Exception as e: # pragma: no cover
log.error(
f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}"
)
@@ -261,7 +259,7 @@ class CurlParser:
# DevTools often uses --data-raw for JSON bodies
# Precedence: --data-binary > --data-raw / -d > --data-urlencode
if parsed_args.data_binary is not None:
if parsed_args.data_binary is not None: # pragma: no cover
try:
data_payload = parsed_args.data_binary.encode("utf-8")
log.debug("Using data from --data-binary as bytes.")
@@ -277,7 +275,7 @@ class CurlParser:
elif parsed_args.data is not None:
data_payload = parsed_args.data
elif parsed_args.data_urlencode:
elif parsed_args.data_urlencode: # pragma: no cover
# Combine and parse urlencoded data
combined_data = "&".join(parsed_args.data_urlencode)
try:
@@ -299,7 +297,7 @@ class CurlParser:
pass # Not JSON, keep it in data_payload
# Handle `-G`: Move data to params if the method is GET
if method == "get" and data_payload:
if method == "get" and data_payload: # pragma: no cover
if isinstance(data_payload, dict): # From --data-urlencode likely
params.update(data_payload)
elif isinstance(data_payload, str):
@@ -369,7 +367,7 @@ class CurlParser:
)
# Ensure request parsing was successful before proceeding
if request is None:
if request is None: # pragma: no cover
log.error("Failed to parse curl command, cannot convert to fetcher.")
return None
@@ -385,22 +383,22 @@ class CurlParser:
try:
return getattr(Fetcher, method)(**request_args)
except Exception as e:
except Exception as e: # pragma: no cover
log.error(f"Error calling Fetcher.{method}: {e}")
return None
else:
else: # pragma: no cover
log.error(
f'Request method "{method}" isn\'t supported by Scrapling yet'
)
return None
else:
else: # pragma: no cover
log.error("Input must be a valid curl command string or a Request object.")
return None
def show_page_in_browser(page: Selector):
def show_page_in_browser(page: Selector): # pragma: no cover
if not page or not isinstance(page, Selector):
log.error("Input must be of type `Selector`")
return
@@ -429,7 +427,7 @@ class CustomShell:
if _known_logging_levels.get(log_level):
self.log_level = _known_logging_levels[log_level]
else:
else: # pragma: no cover
log.warning(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
self.log_level = DEBUG
@@ -480,7 +478,7 @@ class CustomShell:
Type 'exit' or press Ctrl+D to exit.
"""
def update_page(self, result):
def update_page(self, result): # pragma: no cover
"""Update the current page and add to pages history"""
self.page = result
if isinstance(result, (Response, Selector)):
@@ -540,11 +538,11 @@ Type 'exit' or press Ctrl+D to exit.
"help": self.show_help,
}
def show_help(self):
def show_help(self): # pragma: no cover
"""Show help information"""
print(self.banner())
def start(self):
def start(self): # pragma: no cover
"""Start the interactive shell"""
# Get our namespace with application objects
namespace = self.get_namespace()
@@ -594,7 +592,7 @@ class Convertor:
main_content_only: bool = False,
) -> Generator[str, None, None]:
"""Extract the content of a Selector"""
if not page or not isinstance(page, Selector):
if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`")
elif not extraction_type or extraction_type not in cls._extension_map.values():
raise ValueError(f"Unknown extraction type: {extraction_type}")
@@ -627,7 +625,7 @@ class Convertor:
cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None:
"""Write a Selector's content to a file"""
if not page or not isinstance(page, Selector):
if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip():
raise ValueError("Filename must be provided")