refactor: Make all fetchers as an optional dependency group
+ Removing some dead code
This commit is contained in:
Karim shoair
+32
-66
@@ -2,7 +2,6 @@
|
||||
from re import sub as re_sub
|
||||
from sys import stderr
|
||||
from functools import wraps
|
||||
from http import cookies as Cookie
|
||||
from collections import namedtuple
|
||||
from shlex import split as shlex_split
|
||||
from tempfile import mkstemp as make_temp_file
|
||||
@@ -23,25 +22,17 @@ from logging import (
|
||||
from orjson import loads as json_loads, JSONDecodeError
|
||||
|
||||
from scrapling import __version__
|
||||
from scrapling.core.custom_types import TextHandler
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.parser import Selector, Selectors
|
||||
from scrapling.core.custom_types import TextHandler
|
||||
from scrapling.engines.toolbelt.custom import Response
|
||||
from scrapling.core.utils import log, _ParseHeaders, _CookieParser
|
||||
from scrapling.core._types import (
|
||||
List,
|
||||
Optional,
|
||||
Dict,
|
||||
Tuple,
|
||||
Any,
|
||||
extraction_types,
|
||||
Generator,
|
||||
)
|
||||
from scrapling.fetchers import (
|
||||
Fetcher,
|
||||
AsyncFetcher,
|
||||
DynamicFetcher,
|
||||
StealthyFetcher,
|
||||
Response,
|
||||
)
|
||||
|
||||
|
||||
_known_logging_levels = {
|
||||
@@ -71,46 +62,6 @@ Request = namedtuple(
|
||||
)
|
||||
|
||||
|
||||
def _CookieParser(cookie_string):
|
||||
# Errors will be handled on call so the log can be specified
|
||||
cookie_parser = Cookie.SimpleCookie()
|
||||
cookie_parser.load(cookie_string)
|
||||
for key, morsel in cookie_parser.items():
|
||||
yield key, morsel.value
|
||||
|
||||
|
||||
def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:
|
||||
"""Parses headers into separate header and cookie dictionaries."""
|
||||
header_dict = dict()
|
||||
cookie_dict = dict()
|
||||
|
||||
for header_line in header_lines:
|
||||
if ":" not in header_line:
|
||||
if header_line.endswith(";"):
|
||||
header_key = header_line[:-1].strip()
|
||||
header_value = ""
|
||||
header_dict[header_key] = header_value
|
||||
else:
|
||||
raise ValueError(f"Could not parse header without colon: '{header_line}'.")
|
||||
else:
|
||||
header_key, header_value = header_line.split(":", 1)
|
||||
header_key = header_key.strip()
|
||||
header_value = header_value.strip()
|
||||
|
||||
if parse_cookies:
|
||||
if header_key.lower() == "cookie":
|
||||
try:
|
||||
cookie_dict = {key: value for key, value in _CookieParser(header_value)}
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")
|
||||
else:
|
||||
header_dict[header_key] = header_value
|
||||
else:
|
||||
header_dict[header_key] = header_value
|
||||
|
||||
return header_dict, cookie_dict
|
||||
|
||||
|
||||
# Suppress exit on error to handle parsing errors gracefully
|
||||
class NoExitArgumentParser(ArgumentParser): # pragma: no cover
|
||||
def error(self, message):
|
||||
@@ -128,6 +79,9 @@ class CurlParser:
|
||||
"""Builds the argument parser for relevant curl flags from DevTools."""
|
||||
|
||||
def __init__(self):
|
||||
from scrapling.fetchers import Fetcher as __Fetcher
|
||||
|
||||
self.__fetcher = __Fetcher
|
||||
# We will use argparse parser to parse the curl command directly instead of regex
|
||||
# We will focus more on flags that will show up on curl commands copied from DevTools's network tab
|
||||
_parser = NoExitArgumentParser(add_help=False) # Disable default help
|
||||
@@ -343,7 +297,7 @@ class CurlParser:
|
||||
_ = request_args.pop("json", None)
|
||||
|
||||
try:
|
||||
return getattr(Fetcher, method)(**request_args)
|
||||
return getattr(self.__Fetcher, method)(**request_args)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error calling Fetcher.{method}: {e}")
|
||||
return None
|
||||
@@ -377,6 +331,19 @@ class CustomShell:
|
||||
"""A custom IPython shell with minimal dependencies"""
|
||||
|
||||
def __init__(self, code, log_level="debug"):
|
||||
from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed
|
||||
from scrapling.fetchers import (
|
||||
Fetcher as __Fetcher,
|
||||
AsyncFetcher as __AsyncFetcher,
|
||||
DynamicFetcher as __DynamicFetcher,
|
||||
StealthyFetcher as __StealthyFetcher,
|
||||
)
|
||||
|
||||
self.__InteractiveShellEmbed = __InteractiveShellEmbed
|
||||
self.__Fetcher = __Fetcher
|
||||
self.__AsyncFetcher = __AsyncFetcher
|
||||
self.__DynamicFetcher = __DynamicFetcher
|
||||
self.__StealthyFetcher = __StealthyFetcher
|
||||
self.code = code
|
||||
self.page = None
|
||||
self.pages = Selectors([])
|
||||
@@ -400,7 +367,7 @@ class CustomShell:
|
||||
if self.log_level:
|
||||
getLogger("scrapling").setLevel(self.log_level)
|
||||
|
||||
settings = Fetcher.display_config()
|
||||
settings = self.__Fetcher.display_config()
|
||||
settings.pop("storage", None)
|
||||
settings.pop("storage_args", None)
|
||||
log.info(f"Scrapling {__version__} shell started")
|
||||
@@ -466,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit.
|
||||
"""Create a namespace with application-specific objects"""
|
||||
|
||||
# Create wrapped versions of fetch functions
|
||||
get = self.create_wrapper(Fetcher.get)
|
||||
post = self.create_wrapper(Fetcher.post)
|
||||
put = self.create_wrapper(Fetcher.put)
|
||||
delete = self.create_wrapper(Fetcher.delete)
|
||||
dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch)
|
||||
stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)
|
||||
get = self.create_wrapper(self.__Fetcher.get)
|
||||
post = self.create_wrapper(self.__Fetcher.post)
|
||||
put = self.create_wrapper(self.__Fetcher.put)
|
||||
delete = self.create_wrapper(self.__Fetcher.delete)
|
||||
dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)
|
||||
stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch)
|
||||
curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher)
|
||||
|
||||
# Create the namespace dictionary
|
||||
@@ -480,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit.
|
||||
"post": post,
|
||||
"put": put,
|
||||
"delete": delete,
|
||||
"Fetcher": Fetcher,
|
||||
"AsyncFetcher": AsyncFetcher,
|
||||
"Fetcher": self.__Fetcher,
|
||||
"AsyncFetcher": self.__AsyncFetcher,
|
||||
"fetch": dynamic_fetch,
|
||||
"DynamicFetcher": DynamicFetcher,
|
||||
"DynamicFetcher": self.__DynamicFetcher,
|
||||
"stealthy_fetch": stealthy_fetch,
|
||||
"StealthyFetcher": StealthyFetcher,
|
||||
"StealthyFetcher": self.__StealthyFetcher,
|
||||
"Selector": Selector,
|
||||
"page": self.page,
|
||||
"response": self.page,
|
||||
@@ -502,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit.
|
||||
|
||||
def start(self): # pragma: no cover
|
||||
"""Start the interactive shell"""
|
||||
from IPython.terminal.embed import InteractiveShellEmbed
|
||||
|
||||
# Get our namespace with application objects
|
||||
namespace = self.get_namespace()
|
||||
ipython_shell = InteractiveShellEmbed(
|
||||
ipython_shell = self.__InteractiveShellEmbed(
|
||||
banner1=self.banner(),
|
||||
banner2="",
|
||||
enable_tip=False,
|
||||
|
||||
Reference in New Issue
Block a user