refactor: Make all fetchers as an optional dependency group

+ Removing some dead code

scrapling/engines/toolbelt/__init__.py

@@ -5,16 +5,7 @@ from .custom import (
     get_variable_name,
 )
 from .fingerprints import (
-    generate_convincing_referer,
     generate_headers,
     get_os_name,
     __default_useragent__,
 )
-from .navigation import (
-    async_intercept_route,
-    construct_cdp_url,
-    construct_proxy_dict,
-    intercept_route,
-    js_bypass_path,
-)
-from .convertor import ResponseFactory

scrapling/engines/toolbelt/custom.py

@@ -2,8 +2,10 @@
 Functions related to custom types or type checking
 """
 
+from functools import lru_cache
 from email.message import Message
 
+from scrapling.core.utils import log
 from scrapling.core._types import (
     Any,
     Dict,
@@ -12,7 +14,6 @@ from scrapling.core._types import (
     Tuple,
 )
 from scrapling.core.custom_types import MappingProxyType
-from scrapling.core.utils import log, lru_cache
 from scrapling.parser import Selector, SQLiteStorageSystem
 
 

scrapling/engines/toolbelt/fingerprints.py

@@ -2,13 +2,13 @@
 Functions related to generating headers and fingerprints generally
 """
 
+from functools import lru_cache
 from platform import system as platform_system
 
 from tldextract import extract
 from browserforge.headers import Browser, HeaderGenerator
 
 from scrapling.core._types import Dict, Optional
-from scrapling.core.utils import lru_cache
 
 __OS_NAME__ = platform_system()
 
@@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]:
         "Linux": "linux",
         "Darwin": "macos",
         "Windows": "windows",
-        # For the future? because why not?
-        "iOS": "ios",
     }.get(__OS_NAME__)
 
 

scrapling/engines/toolbelt/navigation.py

@@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->
     return None
 
 
-def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
-    """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
-
-    :param cdp_url: The target URL.
-    :param query_params: A dictionary of the parameters to add.
-    :return: The new CDP URL.
-    """
-    try:
-        # Validate the base URL structure
-        parsed = urlparse(cdp_url)
-
-        # Check scheme
-        if parsed.scheme not in ("ws", "wss"):
-            raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
-
-        # Validate hostname and port
-        if not parsed.netloc:
-            raise ValueError("Invalid hostname for the CDP URL")
-
-        try:
-            # Checking if the port is valid (if available)
-            _ = parsed.port
-        except ValueError:
-            # urlparse will raise `ValueError` if the port can't be casted to integer
-            raise ValueError("Invalid port for the CDP URL")
-
-        # Ensure the path starts with /
-        path = parsed.path
-        if not path.startswith("/"):
-            path = "/" + path
-
-        # Reconstruct the base URL with validated parts
-        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
-
-        # Add query parameters
-        if query_params:
-            query_string = urlencode(query_params)
-            return f"{validated_base}?{query_string}"
-
-        return validated_base
-
-    except Exception as e:
-        raise ValueError(f"Invalid CDP URL: {str(e)}")
-
-
 @lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it