freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Make all fetchers as an optional dependency group

Browse Source 
+ Removing some dead code
This commit is contained in:
Karim shoair
2025-09-13 16:07:48 +03:00
parent d7e3deae2a
commit 13d7e70cb7
20 changed files with 142 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/shell.py
+32 -66
View File
@@ -2,7 +2,6 @@
from re import sub as re_sub
from sys import stderr
from functools import wraps
from http import cookies as Cookie
from collections import namedtuple
from shlex import split as shlex_split
from tempfile import mkstemp as make_temp_file
@@ -23,25 +22,17 @@ from logging import (
from orjson import loads as json_loads, JSONDecodeError
from scrapling import __version__
from scrapling.core.custom_types import TextHandler
from scrapling.core.utils import log
from scrapling.parser import Selector, Selectors
from scrapling.core.custom_types import TextHandler
from scrapling.engines.toolbelt.custom import Response
from scrapling.core.utils import log, _ParseHeaders, _CookieParser
from scrapling.core._types import (
List,
Optional,
Dict,
Tuple,
Any,
extraction_types,
Generator,
)
from scrapling.fetchers import (
Fetcher,
AsyncFetcher,
DynamicFetcher,
StealthyFetcher,
Response,
)
_known_logging_levels = {
@@ -71,46 +62,6 @@ Request = namedtuple(
)
def _CookieParser(cookie_string):
# Errors will be handled on call so the log can be specified
cookie_parser = Cookie.SimpleCookie()
cookie_parser.load(cookie_string)
for key, morsel in cookie_parser.items():
yield key, morsel.value
def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Parses headers into separate header and cookie dictionaries."""
header_dict = dict()
cookie_dict = dict()
for header_line in header_lines:
if ":" not in header_line:
if header_line.endswith(";"):
header_key = header_line[:-1].strip()
header_value = ""
header_dict[header_key] = header_value
else:
raise ValueError(f"Could not parse header without colon: '{header_line}'.")
else:
header_key, header_value = header_line.split(":", 1)
header_key = header_key.strip()
header_value = header_value.strip()
if parse_cookies:
if header_key.lower() == "cookie":
try:
cookie_dict = {key: value for key, value in _CookieParser(header_value)}
except Exception as e: # pragma: no cover
raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")
else:
header_dict[header_key] = header_value
else:
header_dict[header_key] = header_value
return header_dict, cookie_dict
# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser): # pragma: no cover
def error(self, message):
@@ -128,6 +79,9 @@ class CurlParser:
"""Builds the argument parser for relevant curl flags from DevTools."""
def __init__(self):
from scrapling.fetchers import Fetcher as __Fetcher
self.__fetcher = __Fetcher
# We will use argparse parser to parse the curl command directly instead of regex
# We will focus more on flags that will show up on curl commands copied from DevTools's network tab
_parser = NoExitArgumentParser(add_help=False) # Disable default help
@@ -343,7 +297,7 @@ class CurlParser:
_ = request_args.pop("json", None)
try:
return getattr(Fetcher, method)(**request_args)
return getattr(self.__Fetcher, method)(**request_args)
except Exception as e: # pragma: no cover
log.error(f"Error calling Fetcher.{method}: {e}")
return None
@@ -377,6 +331,19 @@ class CustomShell:
"""A custom IPython shell with minimal dependencies"""
def __init__(self, code, log_level="debug"):
from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed
from scrapling.fetchers import (
Fetcher as __Fetcher,
AsyncFetcher as __AsyncFetcher,
DynamicFetcher as __DynamicFetcher,
StealthyFetcher as __StealthyFetcher,
)
self.__InteractiveShellEmbed = __InteractiveShellEmbed
self.__Fetcher = __Fetcher
self.__AsyncFetcher = __AsyncFetcher
self.__DynamicFetcher = __DynamicFetcher
self.__StealthyFetcher = __StealthyFetcher
self.code = code
self.page = None
self.pages = Selectors([])
@@ -400,7 +367,7 @@ class CustomShell:
if self.log_level:
getLogger("scrapling").setLevel(self.log_level)
settings = Fetcher.display_config()
settings = self.__Fetcher.display_config()
settings.pop("storage", None)
settings.pop("storage_args", None)
log.info(f"Scrapling {__version__} shell started")
@@ -466,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit.
"""Create a namespace with application-specific objects"""
# Create wrapped versions of fetch functions
get = self.create_wrapper(Fetcher.get)
post = self.create_wrapper(Fetcher.post)
put = self.create_wrapper(Fetcher.put)
delete = self.create_wrapper(Fetcher.delete)
dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch)
stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)
get = self.create_wrapper(self.__Fetcher.get)
post = self.create_wrapper(self.__Fetcher.post)
put = self.create_wrapper(self.__Fetcher.put)
delete = self.create_wrapper(self.__Fetcher.delete)
dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)
stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch)
curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher)
# Create the namespace dictionary
@@ -480,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit.
"post": post,
"put": put,
"delete": delete,
"Fetcher": Fetcher,
"AsyncFetcher": AsyncFetcher,
"Fetcher": self.__Fetcher,
"AsyncFetcher": self.__AsyncFetcher,
"fetch": dynamic_fetch,
"DynamicFetcher": DynamicFetcher,
"DynamicFetcher": self.__DynamicFetcher,
"stealthy_fetch": stealthy_fetch,
"StealthyFetcher": StealthyFetcher,
"StealthyFetcher": self.__StealthyFetcher,
"Selector": Selector,
"page": self.page,
"response": self.page,
@@ -502,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit.
def start(self): # pragma: no cover
"""Start the interactive shell"""
from IPython.terminal.embed import InteractiveShellEmbed
# Get our namespace with application objects
namespace = self.get_namespace()
ipython_shell = InteractiveShellEmbed(
ipython_shell = self.__InteractiveShellEmbed(
banner1=self.banner(),
banner2="",
enable_tip=False,
scrapling/core/translator.py
+2 -2
View File
@@ -10,10 +10,10 @@ So you don't have to learn a new selectors/api method like what bs4 done with so
from functools import lru_cache
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
from scrapling.core._types import Any, Optional, Protocol, Self
scrapling/core/utils/__init__.py
+10
View File
@@ -0,0 +1,10 @@
from ._utils import (
log,
__CONSECUTIVE_SPACES_REGEX__,
flatten,
_is_iterable,
_StorageTools,
clean_spaces,
html_forbidden,
)
from ._shell import _CookieParser, _ParseHeaders
scrapling/core/utils/_shell.py
+48
View File
@@ -0,0 +1,48 @@
from http import cookies as Cookie
from scrapling.core._types import (
List,
Dict,
Tuple,
)
def _CookieParser(cookie_string):
# Errors will be handled on call so the log can be specified
cookie_parser = Cookie.SimpleCookie()
cookie_parser.load(cookie_string)
for key, morsel in cookie_parser.items():
yield key, morsel.value
def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Parses headers into separate header and cookie dictionaries."""
header_dict = dict()
cookie_dict = dict()
for header_line in header_lines:
if ":" not in header_line:
if header_line.endswith(";"):
header_key = header_line[:-1].strip()
header_value = ""
header_dict[header_key] = header_value
else:
raise ValueError(f"Could not parse header without colon: '{header_line}'.")
else:
header_key, header_value = header_line.split(":", 1)
header_key = header_key.strip()
header_value = header_value.strip()
if parse_cookies:
if header_key.lower() == "cookie":
try:
cookie_dict = {key: value for key, value in _CookieParser(header_value)}
except Exception as e: # pragma: no cover
raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")
else:
header_dict[header_key] = header_value
else:
header_dict[header_key] = header_value
return header_dict, cookie_dict
scrapling/core/utils.py → scrapling/core/utils/_utils.py
View File