refactor: Make all fetchers as an optional dependency group

+ Removing some dead code
This commit is contained in:
Karim shoair
2025-09-13 16:07:48 +03:00
parent d7e3deae2a
commit 13d7e70cb7
20 changed files with 142 additions and 174 deletions
-45
View File
@@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->
return None
def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
:param cdp_url: The target URL.
:param query_params: A dictionary of the parameters to add.
:return: The new CDP URL.
"""
try:
# Validate the base URL structure
parsed = urlparse(cdp_url)
# Check scheme
if parsed.scheme not in ("ws", "wss"):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname for the CDP URL")
try:
# Checking if the port is valid (if available)
_ = parsed.port
except ValueError:
# urlparse will raise `ValueError` if the port can't be casted to integer
raise ValueError("Invalid port for the CDP URL")
# Ensure the path starts with /
path = parsed.path
if not path.startswith("/"):
path = "/" + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
# Add query parameters
if query_params:
query_string = urlencode(query_params)
return f"{validated_base}?{query_string}"
return validated_base
except Exception as e:
raise ValueError(f"Invalid CDP URL: {str(e)}")
@lru_cache(10, typed=True)
def js_bypass_path(filename: str) -> str:
"""Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it