refactor: Make all fetchers as an optional dependency group

+ Removing some dead code

scrapling/engines/__init__.py

@@ -1,16 +0,0 @@
-from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS
-from .static import FetcherSession, FetcherClient, AsyncFetcherClient
-from ._browsers import (
-    DynamicSession,
-    AsyncDynamicSession,
-    StealthySession,
-    AsyncStealthySession,
-)
-
-__all__ = [
-    "FetcherSession",
-    "DynamicSession",
-    "AsyncDynamicSession",
-    "StealthySession",
-    "AsyncStealthySession",
-]

scrapling/engines/_browsers/_base.py

@@ -12,20 +12,17 @@ from camoufox.utils import (
     installed_verstr as camoufox_version,
 )
 
-from scrapling.engines.toolbelt import (
-    intercept_route,
-    async_intercept_route,
-    get_os_name,
-)
-from ._page import PageInfo, PagePool
-from ._config_tools import _compiled_stealth_scripts
-from ._validators import validate, PlaywrightConfig, CamoufoxConfig
-from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
 from scrapling.core._types import (
     Any,
     Dict,
     Optional,
 )
+from ._page import PageInfo, PagePool
+from ._config_tools import _compiled_stealth_scripts
+from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.fingerprints import get_os_name
+from ._validators import validate, PlaywrightConfig, CamoufoxConfig
 
 __ff_version_str__ = camoufox_version().split(".", 1)[0]
 

scrapling/engines/_browsers/_camoufox.py

@@ -25,11 +25,11 @@ from scrapling.core._types import (
     Callable,
     SelectorWaitStates,
 )
-from scrapling.engines.toolbelt import (
+from scrapling.engines.toolbelt.convertor import (
     Response,
     ResponseFactory,
-    generate_convincing_referer,
 )
+from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
 
 __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
 _UNSET = object()

scrapling/engines/_browsers/_config_tools.py

@@ -6,7 +6,8 @@ from scrapling.engines.constants import (
     HARMFUL_DEFAULT_ARGS,
     DEFAULT_FLAGS,
 )
-from scrapling.engines.toolbelt import js_bypass_path, generate_headers
+from scrapling.engines.toolbelt.navigation import js_bypass_path
+from scrapling.engines.toolbelt.fingerprints import generate_headers
 
 __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
 

scrapling/engines/_browsers/_controllers.py

@@ -26,11 +26,11 @@ from scrapling.core._types import (
     Callable,
     SelectorWaitStates,
 )
-from scrapling.engines.toolbelt import (
+from scrapling.engines.toolbelt.convertor import (
     Response,
     ResponseFactory,
-    generate_convincing_referer,
 )
+from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
 
 _UNSET = object()
 

scrapling/engines/_browsers/_validators.py

@@ -9,7 +9,7 @@ from scrapling.core._types import (
     List,
     SelectorWaitStates,
 )
-from scrapling.engines.toolbelt import construct_proxy_dict
+from scrapling.engines.toolbelt.navigation import construct_proxy_dict
 
 
 class PlaywrightConfig(Struct, kw_only=True, frozen=False):

scrapling/engines/static.py

@@ -26,11 +26,11 @@ from scrapling.core._types import (
 
 from .toolbelt import (
     Response,
-    generate_convincing_referer,
     generate_headers,
-    ResponseFactory,
     __default_useragent__,
 )
+from .toolbelt.convertor import ResponseFactory
+from .toolbelt.fingerprints import generate_convincing_referer
 
 _UNSET = object()
 

scrapling/engines/toolbelt/__init__.py

@@ -5,16 +5,7 @@ from .custom import (
     get_variable_name,
 )
 from .fingerprints import (
-    generate_convincing_referer,
     generate_headers,
     get_os_name,
     __default_useragent__,
 )
-from .navigation import (
-    async_intercept_route,
-    construct_cdp_url,
-    construct_proxy_dict,
-    intercept_route,
-    js_bypass_path,
-)
-from .convertor import ResponseFactory

scrapling/engines/toolbelt/custom.py

@@ -2,8 +2,10 @@
 Functions related to custom types or type checking
 """
 
+from functools import lru_cache
 from email.message import Message
 
+from scrapling.core.utils import log
 from scrapling.core._types import (
     Any,
     Dict,
@@ -12,7 +14,6 @@ from scrapling.core._types import (
     Tuple,
 )
 from scrapling.core.custom_types import MappingProxyType
-from scrapling.core.utils import log, lru_cache
 from scrapling.parser import Selector, SQLiteStorageSystem
 
 

scrapling/engines/toolbelt/fingerprints.py

@@ -2,13 +2,13 @@
 Functions related to generating headers and fingerprints generally
 """
 
+from functools import lru_cache
 from platform import system as platform_system
 
 from tldextract import extract
 from browserforge.headers import Browser, HeaderGenerator
 
 from scrapling.core._types import Dict, Optional
-from scrapling.core.utils import lru_cache
 
 __OS_NAME__ = platform_system()
 
@@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]:
         "Linux": "linux",
         "Darwin": "macos",
         "Windows": "windows",
-        # For the future? because why not?
-        "iOS": "ios",
     }.get(__OS_NAME__)
 
 

scrapling/engines/toolbelt/navigation.py

@@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->
     return None
 
 
-def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
-    """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
-
-    :param cdp_url: The target URL.
-    :param query_params: A dictionary of the parameters to add.
-    :return: The new CDP URL.
-    """
-    try:
-        # Validate the base URL structure
-        parsed = urlparse(cdp_url)
-
-        # Check scheme
-        if parsed.scheme not in ("ws", "wss"):
-            raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
-
-        # Validate hostname and port
-        if not parsed.netloc:
-            raise ValueError("Invalid hostname for the CDP URL")
-
-        try:
-            # Checking if the port is valid (if available)
-            _ = parsed.port
-        except ValueError:
-            # urlparse will raise `ValueError` if the port can't be casted to integer
-            raise ValueError("Invalid port for the CDP URL")
-
-        # Ensure the path starts with /
-        path = parsed.path
-        if not path.startswith("/"):
-            path = "/" + path
-
-        # Reconstruct the base URL with validated parts
-        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
-
-        # Add query parameters
-        if query_params:
-            query_string = urlencode(query_params)
-            return f"{validated_base}?{query_string}"
-
-        return validated_base
-
-    except Exception as e:
-        raise ValueError(f"Invalid CDP URL: {str(e)}")
-
-
 @lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it