Adding the proxy support to browser-based Fetchers

README.md

@@ -257,6 +257,7 @@ True
 |    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 |       timeout       | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.                                                                                                                                                                                                                                                                                                    |    ✔️    |
 |    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
+|        proxy        | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.                                                                                                                                                                                                                                                                                 |    ✔️    |
 | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 
 </details>
@@ -299,6 +300,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
 | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 |    google_search    | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.                                                                                                                                                                                                                                                                    |    ✔️    |
 |    extra_headers    | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.                                                                                                                                                                                                                                                 |    ✔️    |
+|        proxy        | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.                                                                                                                                                                                                                                                                                 |    ✔️    |
 |     hide_canvas     | Add random noise to canvas operations to prevent fingerprinting.                                                                                                                                                                                                                                                                                                                                                |    ✔️    |
 |    disable_webgl    | Disables WebGL and WebGL 2.0 support entirely.                                                                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |       stealth       | Enables stealth mode, always check the documentation to see what stealth mode does currently.                                                                                                                                                                                                                                                                                                                   |    ✔️    |

scrapling/engines/camo.py

@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
     get_os_name,
     intercept_route,
     check_type_validity,
+    construct_proxy_dict,
     generate_convincing_referer,
 )
 
@@ -18,7 +19,8 @@ class CamoufoxEngine:
             self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
-            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
+            proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
 
@@ -39,6 +41,7 @@ class CamoufoxEngine:
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
         self.headless = headless
@@ -49,6 +52,7 @@ class CamoufoxEngine:
         self.network_idle = bool(network_idle)
         self.google_search = bool(google_search)
         self.extra_headers = extra_headers or {}
+        self.proxy = construct_proxy_dict(proxy)
         self.addons = addons or []
         self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -76,6 +80,7 @@ class CamoufoxEngine:
                 allow_webgl=self.allow_webgl,
                 addons=self.addons,
                 humanize=self.humanize,
+                proxy=self.proxy,
                 i_know_what_im_doing=True,  # To turn warnings off with user configurations
         ) as browser:
             page = browser.new_page()

scrapling/engines/pw.py

@@ -9,8 +9,9 @@ from scrapling.engines.toolbelt import (
     js_bypass_path,
     intercept_route,
     generate_headers,
-    check_type_validity,
     construct_cdp_url,
+    check_type_validity,
+    construct_proxy_dict,
     generate_convincing_referer,
 )
 
@@ -33,6 +34,7 @@ class PlaywrightEngine:
             nstbrowser_config: Optional[Dict] = None,
             google_search: Optional[bool] = True,
             extra_headers: Optional[Dict[str, str]] = None,
+            proxy: Optional[Union[str, Dict[str, str]]] = None,
             adaptor_arguments: Dict = None
     ):
         """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
@@ -54,6 +56,7 @@ class PlaywrightEngine:
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
@@ -65,6 +68,7 @@ class PlaywrightEngine:
         self.disable_webgl = bool(disable_webgl)
         self.google_search = bool(google_search)
         self.extra_headers = extra_headers or {}
+        self.proxy = construct_proxy_dict(proxy)
         self.cdp_url = cdp_url
         self.useragent = useragent
         self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -151,6 +155,7 @@ class PlaywrightEngine:
                     locale='en-US',
                     is_mobile=False,
                     has_touch=False,
+                    proxy=self.proxy,
                     color_scheme='dark',  # Bypasses the 'prefersLightColor' check in creepjs
                     user_agent=useragent,
                     device_scale_factor=2,

scrapling/engines/toolbelt/__init__.py

@@ -15,4 +15,5 @@ from .navigation import (
     js_bypass_path,
     intercept_route,
     construct_cdp_url,
+    construct_proxy_dict,
 )

scrapling/engines/toolbelt/navigation.py

@@ -25,6 +25,40 @@ def intercept_route(route: Route) -> Union[Route, None]:
     return route.continue_()
 
 
+def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
+    """Validate a proxy and return it in the acceptable format for Playwright
+    Reference: https://playwright.dev/python/docs/network#http-proxy
+
+    :param proxy_string: A string or a dictionary representation of the proxy.
+    :return:
+    """
+    if proxy_string:
+        if isinstance(proxy_string, str):
+            proxy = urlparse(proxy_string)
+            try:
+                return {
+                    'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
+                    'username': proxy.username or '',
+                    'password': proxy.password or '',
+                }
+            except ValueError:
+                # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
+                raise TypeError(f'The proxy argument\'s string is in invalid format!')
+
+        elif isinstance(proxy_string, dict):
+            valid_keys = ('server', 'username', 'password', )
+            if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
+                return proxy_string
+            else:
+                raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
+
+        else:
+            raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
+
+    # The default value for proxy in Playwright's source is `None`
+    return None
+
+
 def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
     """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
 

scrapling/fetchers.py

@@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher):
             self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
-            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
@@ -94,23 +94,25 @@ class StealthyFetcher(BaseFetcher):
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
         engine = CamoufoxEngine(
+            proxy=proxy,
+            addons=addons,
             timeout=timeout,
             headless=headless,
-            page_action=page_action,
-            block_images=block_images,
-            block_webrtc=block_webrtc,
-            addons=addons,
             humanize=humanize,
             allow_webgl=allow_webgl,
-            disable_resources=disable_resources,
+            page_action=page_action,
             network_idle=network_idle,
+            block_images=block_images,
+            block_webrtc=block_webrtc,
             wait_selector=wait_selector,
-            wait_selector_state=wait_selector_state,
             google_search=google_search,
             extra_headers=extra_headers,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
             adaptor_arguments=self.adaptor_arguments,
         )
         return engine.fetch(url)
@@ -136,6 +138,7 @@ class PlayWrightFetcher(BaseFetcher):
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
             page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
             hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            proxy: Optional[Union[str, Dict[str, str]]] = None,
             stealth: bool = False,
             cdp_url: Optional[str] = None,
             nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
@@ -157,12 +160,14 @@ class PlayWrightFetcher(BaseFetcher):
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
         engine = PlaywrightEngine(
+            proxy=proxy,
             timeout=timeout,
             stealth=stealth,
             cdp_url=cdp_url,