refactor: Make all fetchers as an optional dependency group
+ Removing some dead code
This commit is contained in:
@@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->
|
||||
return None
|
||||
|
||||
|
||||
def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
|
||||
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
|
||||
|
||||
:param cdp_url: The target URL.
|
||||
:param query_params: A dictionary of the parameters to add.
|
||||
:return: The new CDP URL.
|
||||
"""
|
||||
try:
|
||||
# Validate the base URL structure
|
||||
parsed = urlparse(cdp_url)
|
||||
|
||||
# Check scheme
|
||||
if parsed.scheme not in ("ws", "wss"):
|
||||
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
|
||||
|
||||
# Validate hostname and port
|
||||
if not parsed.netloc:
|
||||
raise ValueError("Invalid hostname for the CDP URL")
|
||||
|
||||
try:
|
||||
# Checking if the port is valid (if available)
|
||||
_ = parsed.port
|
||||
except ValueError:
|
||||
# urlparse will raise `ValueError` if the port can't be casted to integer
|
||||
raise ValueError("Invalid port for the CDP URL")
|
||||
|
||||
# Ensure the path starts with /
|
||||
path = parsed.path
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
|
||||
# Reconstruct the base URL with validated parts
|
||||
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
|
||||
|
||||
# Add query parameters
|
||||
if query_params:
|
||||
query_string = urlencode(query_params)
|
||||
return f"{validated_base}?{query_string}"
|
||||
|
||||
return validated_base
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Invalid CDP URL: {str(e)}")
|
||||
|
||||
|
||||
@lru_cache(10, typed=True)
|
||||
def js_bypass_path(filename: str) -> str:
|
||||
"""Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it
|
||||
|
||||
Reference in New Issue
Block a user