diff --git a/pyproject.toml b/pyproject.toml index a1aa1af..5c708f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,10 @@ dependencies = [ "click>=8.2.1", "orjson>=3.11.3", "tldextract>=5.3.0", +] + +[project.optional-dependencies] +fetchers = [ "curl_cffi>=0.13.0", "playwright>=1.52.0", "rebrowser-playwright>=1.52.0", @@ -68,15 +72,15 @@ dependencies = [ "geoip2>=5.1.0", "msgspec>=0.19.0", ] - -[project.optional-dependencies] ai = [ "mcp>=1.14.0", "markdownify>=1.2.0", + "scrapling[fetchers]", ] shell = [ "IPython>=8.37", # The last version that supports Python 3.10 "markdownify>=1.2.0", + "scrapling[fetchers]", ] all = [ "scrapling[ai,shell]", diff --git a/scrapling/cli.py b/scrapling/cli.py index e4b3c7f..3e052ac 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,11 +2,9 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable -from scrapling.core.utils import log -from scrapling.engines.toolbelt import Response +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _CookieParser, _ParseHeaders from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable -from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher -from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders from orjson import loads as json_loads, JSONDecodeError from click import command, option, Choice, group, argument @@ -40,6 +38,8 @@ def __Request_and_Save( **kwargs, ) -> None: """Make a request using the specified fetcher function and save the result""" + from scrapling.core.shell import Convertor + # Handle relative paths - convert to an absolute path based on the current working directory output_path = Path(output_file) if not output_path.is_absolute(): @@ -251,6 +251,8 @@ def get( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs) @@ -347,6 +349,8 @@ def post( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs) @@ -439,6 +443,8 @@ def put( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs) @@ -524,6 +530,8 @@ def delete( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs) @@ -643,6 +651,8 @@ def fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import DynamicFetcher + __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) @@ -790,6 +800,8 @@ def stealthy_fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import StealthyFetcher + __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 8459a60..2b23b2b 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -2,7 +2,6 @@ from re import sub as re_sub from sys import stderr from functools import wraps -from http import cookies as Cookie from collections import namedtuple from shlex import split as shlex_split from tempfile import mkstemp as make_temp_file @@ -23,25 +22,17 @@ from logging import ( from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ -from scrapling.core.custom_types import TextHandler -from scrapling.core.utils import log from scrapling.parser import Selector, Selectors +from scrapling.core.custom_types import TextHandler +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _ParseHeaders, _CookieParser from scrapling.core._types import ( - List, Optional, Dict, - Tuple, Any, extraction_types, Generator, ) -from scrapling.fetchers import ( - Fetcher, - AsyncFetcher, - DynamicFetcher, - StealthyFetcher, - Response, -) _known_logging_levels = { @@ -71,46 +62,6 @@ Request = namedtuple( ) -def _CookieParser(cookie_string): - # Errors will be handled on call so the log can be specified - cookie_parser = Cookie.SimpleCookie() - cookie_parser.load(cookie_string) - for key, morsel in cookie_parser.items(): - yield key, morsel.value - - -def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: - """Parses headers into separate header and cookie dictionaries.""" - header_dict = dict() - cookie_dict = dict() - - for header_line in header_lines: - if ":" not in header_line: - if header_line.endswith(";"): - header_key = header_line[:-1].strip() - header_value = "" - header_dict[header_key] = header_value - else: - raise ValueError(f"Could not parse header without colon: '{header_line}'.") - else: - header_key, header_value = header_line.split(":", 1) - header_key = header_key.strip() - header_value = header_value.strip() - - if parse_cookies: - if header_key.lower() == "cookie": - try: - cookie_dict = {key: value for key, value in _CookieParser(header_value)} - except Exception as e: # pragma: no cover - raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") - else: - header_dict[header_key] = header_value - else: - header_dict[header_key] = header_value - - return header_dict, cookie_dict - - # Suppress exit on error to handle parsing errors gracefully class NoExitArgumentParser(ArgumentParser): # pragma: no cover def error(self, message): @@ -128,6 +79,9 @@ class CurlParser: """Builds the argument parser for relevant curl flags from DevTools.""" def __init__(self): + from scrapling.fetchers import Fetcher as __Fetcher + + self.__fetcher = __Fetcher # We will use argparse parser to parse the curl command directly instead of regex # We will focus more on flags that will show up on curl commands copied from DevTools's network tab _parser = NoExitArgumentParser(add_help=False) # Disable default help @@ -343,7 +297,7 @@ class CurlParser: _ = request_args.pop("json", None) try: - return getattr(Fetcher, method)(**request_args) + return getattr(self.__Fetcher, method)(**request_args) except Exception as e: # pragma: no cover log.error(f"Error calling Fetcher.{method}: {e}") return None @@ -377,6 +331,19 @@ class CustomShell: """A custom IPython shell with minimal dependencies""" def __init__(self, code, log_level="debug"): + from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed + from scrapling.fetchers import ( + Fetcher as __Fetcher, + AsyncFetcher as __AsyncFetcher, + DynamicFetcher as __DynamicFetcher, + StealthyFetcher as __StealthyFetcher, + ) + + self.__InteractiveShellEmbed = __InteractiveShellEmbed + self.__Fetcher = __Fetcher + self.__AsyncFetcher = __AsyncFetcher + self.__DynamicFetcher = __DynamicFetcher + self.__StealthyFetcher = __StealthyFetcher self.code = code self.page = None self.pages = Selectors([]) @@ -400,7 +367,7 @@ class CustomShell: if self.log_level: getLogger("scrapling").setLevel(self.log_level) - settings = Fetcher.display_config() + settings = self.__Fetcher.display_config() settings.pop("storage", None) settings.pop("storage_args", None) log.info(f"Scrapling {__version__} shell started") @@ -466,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit. """Create a namespace with application-specific objects""" # Create wrapped versions of fetch functions - get = self.create_wrapper(Fetcher.get) - post = self.create_wrapper(Fetcher.post) - put = self.create_wrapper(Fetcher.put) - delete = self.create_wrapper(Fetcher.delete) - dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch) - stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch) + get = self.create_wrapper(self.__Fetcher.get) + post = self.create_wrapper(self.__Fetcher.post) + put = self.create_wrapper(self.__Fetcher.put) + delete = self.create_wrapper(self.__Fetcher.delete) + dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) + stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher) # Create the namespace dictionary @@ -480,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit. "post": post, "put": put, "delete": delete, - "Fetcher": Fetcher, - "AsyncFetcher": AsyncFetcher, + "Fetcher": self.__Fetcher, + "AsyncFetcher": self.__AsyncFetcher, "fetch": dynamic_fetch, - "DynamicFetcher": DynamicFetcher, + "DynamicFetcher": self.__DynamicFetcher, "stealthy_fetch": stealthy_fetch, - "StealthyFetcher": StealthyFetcher, + "StealthyFetcher": self.__StealthyFetcher, "Selector": Selector, "page": self.page, "response": self.page, @@ -502,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit. def start(self): # pragma: no cover """Start the interactive shell""" - from IPython.terminal.embed import InteractiveShellEmbed # Get our namespace with application objects namespace = self.get_namespace() - ipython_shell = InteractiveShellEmbed( + ipython_shell = self.__InteractiveShellEmbed( banner1=self.banner(), banner2="", enable_tip=False, diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 88dfbff..d98092e 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -10,10 +10,10 @@ So you don't have to learn a new selectors/api method like what bs4 done with so from functools import lru_cache -from cssselect import HTMLTranslator as OriginalHTMLTranslator -from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from cssselect.xpath import ExpressionError from cssselect.xpath import XPathExpr as OriginalXPathExpr +from cssselect import HTMLTranslator as OriginalHTMLTranslator +from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from scrapling.core._types import Any, Optional, Protocol, Self diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.py new file mode 100644 index 0000000..dc95705 --- /dev/null +++ b/scrapling/core/utils/__init__.py @@ -0,0 +1,10 @@ +from ._utils import ( + log, + __CONSECUTIVE_SPACES_REGEX__, + flatten, + _is_iterable, + _StorageTools, + clean_spaces, + html_forbidden, +) +from ._shell import _CookieParser, _ParseHeaders diff --git a/scrapling/core/utils/_shell.py b/scrapling/core/utils/_shell.py new file mode 100644 index 0000000..05420ae --- /dev/null +++ b/scrapling/core/utils/_shell.py @@ -0,0 +1,48 @@ +from http import cookies as Cookie + + +from scrapling.core._types import ( + List, + Dict, + Tuple, +) + + +def _CookieParser(cookie_string): + # Errors will be handled on call so the log can be specified + cookie_parser = Cookie.SimpleCookie() + cookie_parser.load(cookie_string) + for key, morsel in cookie_parser.items(): + yield key, morsel.value + + +def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: + """Parses headers into separate header and cookie dictionaries.""" + header_dict = dict() + cookie_dict = dict() + + for header_line in header_lines: + if ":" not in header_line: + if header_line.endswith(";"): + header_key = header_line[:-1].strip() + header_value = "" + header_dict[header_key] = header_value + else: + raise ValueError(f"Could not parse header without colon: '{header_line}'.") + else: + header_key, header_value = header_line.split(":", 1) + header_key = header_key.strip() + header_value = header_value.strip() + + if parse_cookies: + if header_key.lower() == "cookie": + try: + cookie_dict = {key: value for key, value in _CookieParser(header_value)} + except Exception as e: # pragma: no cover + raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") + else: + header_dict[header_key] = header_value + else: + header_dict[header_key] = header_value + + return header_dict, cookie_dict diff --git a/scrapling/core/utils.py b/scrapling/core/utils/_utils.py similarity index 100% rename from scrapling/core/utils.py rename to scrapling/core/utils/_utils.py diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index 477bb21..e69de29 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,16 +0,0 @@ -from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS -from .static import FetcherSession, FetcherClient, AsyncFetcherClient -from ._browsers import ( - DynamicSession, - AsyncDynamicSession, - StealthySession, - AsyncStealthySession, -) - -__all__ = [ - "FetcherSession", - "DynamicSession", - "AsyncDynamicSession", - "StealthySession", - "AsyncStealthySession", -] diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 5bb86ab..00c5567 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -12,20 +12,17 @@ from camoufox.utils import ( installed_verstr as camoufox_version, ) -from scrapling.engines.toolbelt import ( - intercept_route, - async_intercept_route, - get_os_name, -) -from ._page import PageInfo, PagePool -from ._config_tools import _compiled_stealth_scripts -from ._validators import validate, PlaywrightConfig, CamoufoxConfig -from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route from scrapling.core._types import ( Any, Dict, Optional, ) +from ._page import PageInfo, PagePool +from ._config_tools import _compiled_stealth_scripts +from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.fingerprints import get_os_name +from ._validators import validate, PlaywrightConfig, CamoufoxConfig __ff_version_str__ = camoufox_version().split(".", 1)[0] diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 31113be..6e0bc1e 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -25,11 +25,11 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - generate_convincing_referer, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") _UNSET = object() diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 96b7f41..4b405e6 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -6,7 +6,8 @@ from scrapling.engines.constants import ( HARMFUL_DEFAULT_ARGS, DEFAULT_FLAGS, ) -from scrapling.engines.toolbelt import js_bypass_path, generate_headers +from scrapling.engines.toolbelt.navigation import js_bypass_path +from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 96d7291..07efee2 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -26,11 +26,11 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - generate_convincing_referer, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer _UNSET = object() diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index a20ae55..ca64942 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -9,7 +9,7 @@ from scrapling.core._types import ( List, SelectorWaitStates, ) -from scrapling.engines.toolbelt import construct_proxy_dict +from scrapling.engines.toolbelt.navigation import construct_proxy_dict class PlaywrightConfig(Struct, kw_only=True, frozen=False): diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index e028feb..35271b8 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -26,11 +26,11 @@ from scrapling.core._types import ( from .toolbelt import ( Response, - generate_convincing_referer, generate_headers, - ResponseFactory, __default_useragent__, ) +from .toolbelt.convertor import ResponseFactory +from .toolbelt.fingerprints import generate_convincing_referer _UNSET = object() diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index d58fd57..04f43b1 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -5,16 +5,7 @@ from .custom import ( get_variable_name, ) from .fingerprints import ( - generate_convincing_referer, generate_headers, get_os_name, __default_useragent__, ) -from .navigation import ( - async_intercept_route, - construct_cdp_url, - construct_proxy_dict, - intercept_route, - js_bypass_path, -) -from .convertor import ResponseFactory diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 72c39e9..f9effa2 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -2,8 +2,10 @@ Functions related to custom types or type checking """ +from functools import lru_cache from email.message import Message +from scrapling.core.utils import log from scrapling.core._types import ( Any, Dict, @@ -12,7 +14,6 @@ from scrapling.core._types import ( Tuple, ) from scrapling.core.custom_types import MappingProxyType -from scrapling.core.utils import log, lru_cache from scrapling.parser import Selector, SQLiteStorageSystem diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 4ca8e38..bd836e7 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -2,13 +2,13 @@ Functions related to generating headers and fingerprints generally """ +from functools import lru_cache from platform import system as platform_system from tldextract import extract from browserforge.headers import Browser, HeaderGenerator from scrapling.core._types import Dict, Optional -from scrapling.core.utils import lru_cache __OS_NAME__ = platform_system() @@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]: "Linux": "linux", "Darwin": "macos", "Windows": "windows", - # For the future? because why not? - "iOS": "ios", }.get(__OS_NAME__) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 6d666a8..f2f445c 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> return None -def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: - """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists - - :param cdp_url: The target URL. - :param query_params: A dictionary of the parameters to add. - :return: The new CDP URL. - """ - try: - # Validate the base URL structure - parsed = urlparse(cdp_url) - - # Check scheme - if parsed.scheme not in ("ws", "wss"): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") - - # Validate hostname and port - if not parsed.netloc: - raise ValueError("Invalid hostname for the CDP URL") - - try: - # Checking if the port is valid (if available) - _ = parsed.port - except ValueError: - # urlparse will raise `ValueError` if the port can't be casted to integer - raise ValueError("Invalid port for the CDP URL") - - # Ensure the path starts with / - path = parsed.path - if not path.startswith("/"): - path = "/" + path - - # Reconstruct the base URL with validated parts - validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" - - # Add query parameters - if query_params: - query_string = urlencode(query_params) - return f"{validated_base}?{query_string}" - - return validated_base - - except Exception as e: - raise ValueError(f"Invalid CDP URL: {str(e)}") - - @lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 2421b42..323293d 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -6,15 +6,17 @@ from scrapling.core._types import ( SelectorWaitStates, Iterable, ) -from scrapling.engines import ( +from scrapling.engines.static import ( FetcherSession, - StealthySession, - AsyncStealthySession, - DynamicSession, - AsyncDynamicSession, FetcherClient as _FetcherClient, AsyncFetcherClient as _AsyncFetcherClient, ) +from scrapling.engines._browsers import ( + DynamicSession, + StealthySession, + AsyncDynamicSession, + AsyncStealthySession, +) from scrapling.engines.toolbelt import BaseFetcher, Response __FetcherClientInstance__ = _FetcherClient() diff --git a/scrapling/parser.py b/scrapling/parser.py index f943db8..5335a82 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,12 +1,11 @@ -from pathlib import Path import re +from pathlib import Path from inspect import signature -from difflib import SequenceMatcher from urllib.parse import urljoin +from difflib import SequenceMatcher -from cssselect import SelectorError, SelectorSyntaxError -from cssselect import parse as split_selectors from lxml.html import HtmlElement, HtmlMixin, HTMLParser +from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors from lxml.etree import ( XPath, tostring,