freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding the proxy support to browser-based Fetchers

Browse Source
This commit is contained in:
Karim shoair
2024-11-15 14:43:03 +02:00
parent b993adb4a7
commit 2abb70218f
6 changed files with 61 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -257,6 +257,7 @@ True
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
</details>
@@ -299,6 +300,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ |
| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ |
| disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ |
| stealth | Enables stealth mode, always check the documentation to see what stealth mode does currently. | ✔️ |
scrapling/engines/camo.py
+6 -1
View File
@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
get_os_name,
intercept_route,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
)
@@ -18,7 +19,8 @@ class CamoufoxEngine:
self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None
):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -39,6 +41,7 @@ class CamoufoxEngine:
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
self.headless = headless
@@ -49,6 +52,7 @@ class CamoufoxEngine:
self.network_idle = bool(network_idle)
self.google_search = bool(google_search)
self.extra_headers = extra_headers or {}
self.proxy = construct_proxy_dict(proxy)
self.addons = addons or []
self.humanize = humanize
self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -76,6 +80,7 @@ class CamoufoxEngine:
allow_webgl=self.allow_webgl,
addons=self.addons,
humanize=self.humanize,
proxy=self.proxy,
i_know_what_im_doing=True, # To turn warnings off with user configurations
) as browser:
page = browser.new_page()
scrapling/engines/pw.py
+6 -1
View File
@@ -9,8 +9,9 @@ from scrapling.engines.toolbelt import (
js_bypass_path,
intercept_route,
generate_headers,
check_type_validity,
construct_cdp_url,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
)
@@ -33,6 +34,7 @@ class PlaywrightEngine:
nstbrowser_config: Optional[Dict] = None,
google_search: Optional[bool] = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None
):
"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
@@ -54,6 +56,7 @@ class PlaywrightEngine:
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
@@ -65,6 +68,7 @@ class PlaywrightEngine:
self.disable_webgl = bool(disable_webgl)
self.google_search = bool(google_search)
self.extra_headers = extra_headers or {}
self.proxy = construct_proxy_dict(proxy)
self.cdp_url = cdp_url
self.useragent = useragent
self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -151,6 +155,7 @@ class PlaywrightEngine:
locale='en-US',
is_mobile=False,
has_touch=False,
proxy=self.proxy,
color_scheme='dark', # Bypasses the 'prefersLightColor' check in creepjs
user_agent=useragent,
device_scale_factor=2,
scrapling/engines/toolbelt/__init__.py
+1
View File
@@ -15,4 +15,5 @@ from .navigation import (
js_bypass_path,
intercept_route,
construct_cdp_url,
construct_proxy_dict,
)
scrapling/engines/toolbelt/navigation.py
+34
View File
@@ -25,6 +25,40 @@ def intercept_route(route: Route) -> Union[Route, None]:
return route.continue_()
def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy
:param proxy_string: A string or a dictionary representation of the proxy.
:return:
"""
if proxy_string:
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
try:
return {
'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
'username': proxy.username or '',
'password': proxy.password or '',
}
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError(f'The proxy argument\'s string is in invalid format!')
elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', )
if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
return proxy_string
else:
raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
else:
raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
# The default value for proxy in Playwright's source is `None`
return None
def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
scrapling/fetchers.py
+12 -7
View File
@@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher):
self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,
) -> Response:
"""
Opens up a browser and do your request based on your chosen options below.
@@ -94,23 +94,25 @@ class StealthyFetcher(BaseFetcher):
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
"""
engine = CamoufoxEngine(
proxy=proxy,
addons=addons,
timeout=timeout,
headless=headless,
page_action=page_action,
block_images=block_images,
block_webrtc=block_webrtc,
addons=addons,
humanize=humanize,
allow_webgl=allow_webgl,
disable_resources=disable_resources,
page_action=page_action,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,
wait_selector=wait_selector,
wait_selector_state=wait_selector_state,
google_search=google_search,
extra_headers=extra_headers,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
@@ -136,6 +138,7 @@ class PlayWrightFetcher(BaseFetcher):
useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
proxy: Optional[Union[str, Dict[str, str]]] = None,
stealth: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
@@ -157,12 +160,14 @@ class PlayWrightFetcher(BaseFetcher):
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
"""
engine = PlaywrightEngine(
proxy=proxy,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,