freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PlaywrightEngine - Validate CDP URLs in all cases

Browse Source
This commit is contained in:
Karim shoair
2024-11-06 11:24:56 +02:00
parent 6497b1b559
commit 6cf506cb22
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/pw.py
+3
View File
@@ -96,6 +96,9 @@ class PlaywrightEngine:
# 'token': '' # 'token': ''
} }
cdp_url = construct_cdp_url(cdp_url, config) cdp_url = construct_cdp_url(cdp_url, config)
else:
# To validate it
cdp_url = construct_cdp_url(cdp_url)
return cdp_url return cdp_url
scrapling/engines/toolbelt/navigation.py
+2 -2
View File
@@ -6,7 +6,7 @@ import os
import logging import logging
from urllib.parse import urlparse, urlencode from urllib.parse import urlparse, urlencode
from scrapling.core._types import Union, Dict from scrapling.core._types import Union, Dict, Optional
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
from playwright.sync_api import Route from playwright.sync_api import Route
@@ -24,7 +24,7 @@ def intercept_route(route: Route) -> Union[Route, None]:
return route.continue_() return route.continue_()
def construct_cdp_url(cdp_url: str, query_params: Dict) -> str: def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
:param cdp_url: The target URL. :param cdp_url: The target URL.