diff --git a/README.md b/README.md index 33ebbf9..d19981e 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,7 @@ True | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | | timeout | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | +| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | @@ -299,6 +300,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | | google_search | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ | +| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | | hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ | | disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ | | stealth | Enables stealth mode, always check the documentation to see what stealth mode does currently. | ✔️ | diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 3677531..6a26495 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import ( get_os_name, intercept_route, check_type_validity, + construct_proxy_dict, generate_convincing_referer, ) @@ -18,7 +19,8 @@ class CamoufoxEngine: self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False, block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None, - wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None + wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, + proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None ): """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation. @@ -39,6 +41,7 @@ class CamoufoxEngine: :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. """ self.headless = headless @@ -49,6 +52,7 @@ class CamoufoxEngine: self.network_idle = bool(network_idle) self.google_search = bool(google_search) self.extra_headers = extra_headers or {} + self.proxy = construct_proxy_dict(proxy) self.addons = addons or [] self.humanize = humanize self.timeout = check_type_validity(timeout, [int, float], 30000) @@ -76,6 +80,7 @@ class CamoufoxEngine: allow_webgl=self.allow_webgl, addons=self.addons, humanize=self.humanize, + proxy=self.proxy, i_know_what_im_doing=True, # To turn warnings off with user configurations ) as browser: page = browser.new_page() diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 2d6ebf2..b97bcac 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -9,8 +9,9 @@ from scrapling.engines.toolbelt import ( js_bypass_path, intercept_route, generate_headers, - check_type_validity, construct_cdp_url, + check_type_validity, + construct_proxy_dict, generate_convincing_referer, ) @@ -33,6 +34,7 @@ class PlaywrightEngine: nstbrowser_config: Optional[Dict] = None, google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, + proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None ): """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation. @@ -54,6 +56,7 @@ class PlaywrightEngine: :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config. :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. """ @@ -65,6 +68,7 @@ class PlaywrightEngine: self.disable_webgl = bool(disable_webgl) self.google_search = bool(google_search) self.extra_headers = extra_headers or {} + self.proxy = construct_proxy_dict(proxy) self.cdp_url = cdp_url self.useragent = useragent self.timeout = check_type_validity(timeout, [int, float], 30000) @@ -151,6 +155,7 @@ class PlaywrightEngine: locale='en-US', is_mobile=False, has_touch=False, + proxy=self.proxy, color_scheme='dark', # Bypasses the 'prefersLightColor' check in creepjs user_agent=useragent, device_scale_factor=2, diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index 08b559b..ac3e03d 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -15,4 +15,5 @@ from .navigation import ( js_bypass_path, intercept_route, construct_cdp_url, + construct_proxy_dict, ) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index cf73a39..363f233 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -25,6 +25,40 @@ def intercept_route(route: Route) -> Union[Route, None]: return route.continue_() +def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]: + """Validate a proxy and return it in the acceptable format for Playwright + Reference: https://playwright.dev/python/docs/network#http-proxy + + :param proxy_string: A string or a dictionary representation of the proxy. + :return: + """ + if proxy_string: + if isinstance(proxy_string, str): + proxy = urlparse(proxy_string) + try: + return { + 'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}', + 'username': proxy.username or '', + 'password': proxy.password or '', + } + except ValueError: + # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc... + raise TypeError(f'The proxy argument\'s string is in invalid format!') + + elif isinstance(proxy_string, dict): + valid_keys = ('server', 'username', 'password', ) + if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()): + return proxy_string + else: + raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}') + + else: + raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!') + + # The default value for proxy in Playwright's source is `None` + return None + + def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 65a8901..b0f1d60 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher): self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False, block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, - wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None + wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None, ) -> Response: """ Opens up a browser and do your request based on your chosen options below. @@ -94,23 +94,25 @@ class StealthyFetcher(BaseFetcher): :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. """ engine = CamoufoxEngine( + proxy=proxy, + addons=addons, timeout=timeout, headless=headless, - page_action=page_action, - block_images=block_images, - block_webrtc=block_webrtc, - addons=addons, humanize=humanize, allow_webgl=allow_webgl, - disable_resources=disable_resources, + page_action=page_action, network_idle=network_idle, + block_images=block_images, + block_webrtc=block_webrtc, wait_selector=wait_selector, - wait_selector_state=wait_selector_state, google_search=google_search, extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, adaptor_arguments=self.adaptor_arguments, ) return engine.fetch(url) @@ -136,6 +138,7 @@ class PlayWrightFetcher(BaseFetcher): useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached', hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True, + proxy: Optional[Union[str, Dict[str, str]]] = None, stealth: bool = False, cdp_url: Optional[str] = None, nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, @@ -157,12 +160,14 @@ class PlayWrightFetcher(BaseFetcher): :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored. :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config. :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. """ engine = PlaywrightEngine( + proxy=proxy, timeout=timeout, stealth=stealth, cdp_url=cdp_url,