From 4947152be2f3a57b1f957532a8fcb66d488fb1ca Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 1 Jan 2026 21:06:35 +0200 Subject: [PATCH] docs: change docstring style for some methods --- scrapling/engines/_browsers/_controllers.py | 50 ++++----- scrapling/engines/_browsers/_stealth.py | 54 +++++---- scrapling/fetchers/chrome.py | 98 ++++++++-------- scrapling/fetchers/stealth_chrome.py | 118 ++++++++++---------- 4 files changed, 156 insertions(+), 164 deletions(-) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 065aad3..ff21438 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -89,19 +89,18 @@ class DynamicSession(SyncSession, DynamicSessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param kwargs: Additional keyword arguments including: - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ params = _validate(kwargs, self, PlaywrightConfig) @@ -219,19 +218,18 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param kwargs: Additional keyword arguments including: - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ params = _validate(kwargs, self, PlaywrightConfig) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 7b05c1b..912d38e 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -175,20 +175,19 @@ class StealthySession(SyncSession, StealthySessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param kwargs: Additional keyword arguments including: - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ params = _validate(kwargs, self, StealthConfig) @@ -388,20 +387,19 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param kwargs: Additional keyword arguments including: - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ params = _validate(kwargs, self, StealthConfig) diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index c572d59..0d50c19 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -12,31 +12,30 @@ class DynamicFetcher(BaseFetcher): """Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param kwargs: Browser session configuration options including: - - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - disable_resources: Drop requests of unnecessary resources for a speed boost. - - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - - cookies: Set cookies for the next request. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - - locale: Set the locale for the browser if wanted. The default value is `en-US`. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. - - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - - disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. - - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - extra_flags: A list of additional browser flags to pass to the browser on launch. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Playwright's context as additional settings. + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param disable_resources: Drop requests of unnecessary resources for a speed boost. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + :param cookies: Set cookies for the next request. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param extra_headers: A dictionary of extra headers to add to the request. + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get( @@ -55,31 +54,30 @@ class DynamicFetcher(BaseFetcher): """Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param kwargs: Browser session configuration options including: - - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - disable_resources: Drop requests of unnecessary resources for a speed boost. - - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - - cookies: Set cookies for the next request. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - - locale: Set the locale for the browser if wanted. The default value is `en-US`. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. - - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - - disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. - - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - extra_flags: A list of additional browser flags to pass to the browser on launch. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Playwright's context as additional settings. + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param disable_resources: Drop requests of unnecessary resources for a speed boost. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + :param cookies: Set cookies for the next request. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param extra_headers: A dictionary of extra headers to add to the request. + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get( diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py index 64511a2..58574d3 100644 --- a/scrapling/fetchers/stealth_chrome.py +++ b/scrapling/fetchers/stealth_chrome.py @@ -16,36 +16,35 @@ class StealthyFetcher(BaseFetcher): Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param kwargs: Browser session configuration options including: - - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - - cookies: Set cookies for the next request. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - - locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting - rules. Defaults to the system default locale. - - timezone_id: Changes the timezone of the browser. Defaults to the system timezone. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - - block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. - - allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. - - extra_flags: A list of additional browser flags to pass to the browser on launch. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + :param cookies: Set cookies for the next request. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get( @@ -65,36 +64,35 @@ class StealthyFetcher(BaseFetcher): Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param kwargs: Browser session configuration options including: - - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - disable_resources: Drop requests for unnecessary resources for a speed boost. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - - cookies: Set cookies for the next request. - - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - - locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting - rules. Defaults to the system default locale. - - timezone_id: Changes the timezone of the browser. Defaults to the system timezone. - - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - - block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. - - allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. - - extra_flags: A list of additional browser flags to pass to the browser on launch. - - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + :param cookies: Set cookies for the next request. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get(