refactor(fetchers)!: Replace Camoufox with patchright and many optimizations
- DynamicFetcher became 20% faster - StealthyFetcher became 99% faster - Scrapling size decreased - Code became ~400 lines shorter - Most importantly, scrapling is more stable and reliable now. - Less confusing for new users. - More...
This commit is contained in:
Karim shoair
@@ -1,7 +1,6 @@
|
||||
from time import time
|
||||
from asyncio import sleep as asyncio_sleep, Lock
|
||||
|
||||
from camoufox import DefaultAddons
|
||||
from playwright.sync_api._generated import Page
|
||||
from playwright.sync_api import (
|
||||
Frame,
|
||||
@@ -17,18 +16,18 @@ from playwright.async_api import (
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
)
|
||||
from playwright._impl._errors import Error as PlaywrightError
|
||||
from camoufox.pkgman import installed_verstr as camoufox_version
|
||||
from camoufox.utils import launch_options as generate_launch_options
|
||||
|
||||
from ._page import PageInfo, PagePool
|
||||
from scrapling.parser import Selector
|
||||
from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING
|
||||
from scrapling.engines.toolbelt.fingerprints import get_os_name
|
||||
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
|
||||
from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
|
||||
from ._validators import validate, PlaywrightConfig, StealthConfig
|
||||
from ._config_tools import __default_chrome_useragent__, __default_useragent__
|
||||
from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
|
||||
|
||||
__ff_version_str__ = camoufox_version().split(".", 1)[0]
|
||||
from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING, overload, Tuple
|
||||
from scrapling.engines.constants import (
|
||||
DEFAULT_STEALTH_FLAGS,
|
||||
HARMFUL_DEFAULT_ARGS,
|
||||
DEFAULT_FLAGS,
|
||||
)
|
||||
|
||||
|
||||
class SyncSession:
|
||||
@@ -84,10 +83,6 @@ class SyncSession:
|
||||
if disable_resources:
|
||||
page.route("**/*", intercept_route)
|
||||
|
||||
if getattr(self, "stealth", False):
|
||||
for script in _compiled_stealth_scripts():
|
||||
page.add_init_script(script=script)
|
||||
|
||||
page_info = self.page_pool.add_page(page)
|
||||
page_info.mark_busy()
|
||||
return page_info
|
||||
@@ -202,10 +197,6 @@ class AsyncSession:
|
||||
if disable_resources:
|
||||
await page.route("**/*", async_intercept_route)
|
||||
|
||||
if getattr(self, "stealth", False):
|
||||
for script in _compiled_stealth_scripts():
|
||||
await page.add_init_script(script=script)
|
||||
|
||||
return self.page_pool.add_page(page)
|
||||
|
||||
def get_pool_stats(self) -> Dict[str, int]:
|
||||
@@ -251,151 +242,118 @@ class AsyncSession:
|
||||
return handle_response
|
||||
|
||||
|
||||
class DynamicSessionMixin:
|
||||
def __validate__(self, **params):
|
||||
class BaseSessionMixin:
|
||||
@overload
|
||||
def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...
|
||||
|
||||
@overload
|
||||
def __validate_routine__(self, params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
|
||||
|
||||
def __validate_routine__(
|
||||
self, params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]
|
||||
) -> PlaywrightConfig | StealthConfig:
|
||||
# Dark color scheme bypasses the 'prefersLightColor' check in creepjs
|
||||
self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
|
||||
self._launch_options: Dict[str, Any] = self._context_options | {
|
||||
"args": DEFAULT_FLAGS,
|
||||
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
|
||||
}
|
||||
if "__max_pages" in params:
|
||||
params["max_pages"] = params.pop("__max_pages")
|
||||
|
||||
config = validate(params, model=PlaywrightConfig)
|
||||
config = validate(params, model=model)
|
||||
self._headers_keys = (
|
||||
{header.lower() for header in config.extra_headers.keys()} if config.extra_headers else set()
|
||||
)
|
||||
|
||||
self._max_pages = config.max_pages
|
||||
self._headless = config.headless
|
||||
self._hide_canvas = config.hide_canvas
|
||||
self._disable_webgl = config.disable_webgl
|
||||
self._real_chrome = config.real_chrome
|
||||
self._stealth = config.stealth
|
||||
self._google_search = config.google_search
|
||||
self._wait = config.wait
|
||||
self._proxy = config.proxy
|
||||
self._locale = config.locale
|
||||
self._extra_headers = config.extra_headers
|
||||
self._useragent = config.useragent
|
||||
self._timeout = config.timeout
|
||||
self._cookies = config.cookies
|
||||
self._disable_resources = config.disable_resources
|
||||
self._cdp_url = config.cdp_url
|
||||
self._network_idle = config.network_idle
|
||||
self._load_dom = config.load_dom
|
||||
self._wait_selector = config.wait_selector
|
||||
self._init_script = config.init_script
|
||||
self._wait_selector_state = config.wait_selector_state
|
||||
self._extra_flags = config.extra_flags
|
||||
self._selector_config = config.selector_config
|
||||
self._timezone_id = config.timezone_id
|
||||
self._additional_args = config.additional_args
|
||||
self._page_action = config.page_action
|
||||
self._user_data_dir = config.user_data_dir
|
||||
self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()
|
||||
self.__initiate_browser_options__()
|
||||
return config
|
||||
|
||||
def __initiate_browser_options__(self):
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(self._proxy, tuple)
|
||||
|
||||
if not self._cdp_url:
|
||||
# `launch_options` is used with persistent context
|
||||
self.launch_options = dict(
|
||||
_launch_kwargs(
|
||||
self._headless,
|
||||
self._proxy,
|
||||
self._locale,
|
||||
tuple(self._extra_headers.items()) if self._extra_headers else tuple(),
|
||||
self._useragent,
|
||||
self._real_chrome,
|
||||
self._stealth,
|
||||
self._hide_canvas,
|
||||
self._disable_webgl,
|
||||
self._timezone_id,
|
||||
tuple(self._extra_flags) if self._extra_flags else tuple(),
|
||||
)
|
||||
)
|
||||
self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
|
||||
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
|
||||
self.launch_options["user_data_dir"] = self._user_data_dir
|
||||
self.launch_options.update(cast(Dict, self._additional_args))
|
||||
self.context_options = dict()
|
||||
else:
|
||||
# while `context_options` is left to be used when cdp mode is enabled
|
||||
self.launch_options = dict()
|
||||
self.context_options = dict(
|
||||
_context_kwargs(
|
||||
self._proxy,
|
||||
self._locale,
|
||||
tuple(self._extra_headers.items()) if self._extra_headers else tuple(),
|
||||
self._useragent,
|
||||
self._stealth,
|
||||
)
|
||||
)
|
||||
self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])
|
||||
self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
|
||||
self.context_options.update(cast(Dict, self._additional_args))
|
||||
|
||||
|
||||
class StealthySessionMixin:
|
||||
def __validate__(self, **params):
|
||||
if "__max_pages" in params:
|
||||
params["max_pages"] = params.pop("__max_pages")
|
||||
|
||||
config: CamoufoxConfig = validate(params, model=CamoufoxConfig)
|
||||
|
||||
self._max_pages = config.max_pages
|
||||
self._headless = config.headless
|
||||
self._block_images = config.block_images
|
||||
self._disable_resources = config.disable_resources
|
||||
self._block_webrtc = config.block_webrtc
|
||||
self._allow_webgl = config.allow_webgl
|
||||
self._network_idle = config.network_idle
|
||||
self._load_dom = config.load_dom
|
||||
self._humanize = config.humanize
|
||||
self._solve_cloudflare = config.solve_cloudflare
|
||||
self._wait = config.wait
|
||||
self._timeout = config.timeout
|
||||
self._page_action = config.page_action
|
||||
self._wait_selector = config.wait_selector
|
||||
self._init_script = config.init_script
|
||||
self._addons = config.addons
|
||||
self._wait_selector_state = config.wait_selector_state
|
||||
self._cookies = config.cookies
|
||||
self._google_search = config.google_search
|
||||
self._extra_headers = config.extra_headers
|
||||
self._proxy = config.proxy
|
||||
self._os_randomize = config.os_randomize
|
||||
self._disable_ads = config.disable_ads
|
||||
self._geoip = config.geoip
|
||||
self._selector_config = config.selector_config
|
||||
self._additional_args = config.additional_args
|
||||
self._user_data_dir = config.user_data_dir
|
||||
self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()
|
||||
self.__initiate_browser_options__()
|
||||
|
||||
def __initiate_browser_options__(self):
|
||||
"""Initiate browser options."""
|
||||
self.launch_options: Dict[str, Any] = generate_launch_options(
|
||||
**{
|
||||
"geoip": self._geoip,
|
||||
"proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy,
|
||||
"addons": self._addons,
|
||||
"exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO],
|
||||
"headless": self._headless,
|
||||
"humanize": True if self._solve_cloudflare else self._humanize,
|
||||
"i_know_what_im_doing": True, # To turn warnings off with the user configurations
|
||||
"allow_webgl": self._allow_webgl,
|
||||
"block_webrtc": self._block_webrtc,
|
||||
"block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
|
||||
"os": None if self._os_randomize else get_os_name(),
|
||||
"user_data_dir": self._user_data_dir,
|
||||
"ff_version": __ff_version_str__,
|
||||
"firefox_user_prefs": {
|
||||
# This is what enabling `enable_cache` does internally, so we do it from here instead
|
||||
"browser.sessionhistory.max_entries": 10,
|
||||
"browser.sessionhistory.max_total_viewers": -1,
|
||||
"browser.cache.memory.enable": True,
|
||||
"browser.cache.disk_cache_ssl": True,
|
||||
"browser.cache.disk.smart_size.enabled": True,
|
||||
},
|
||||
**cast(Dict, self._additional_args),
|
||||
def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
|
||||
config = cast(PlaywrightConfig, getattr(self, "_config", None))
|
||||
self._context_options.update(
|
||||
{
|
||||
"proxy": config.proxy,
|
||||
"locale": config.locale,
|
||||
"timezone_id": config.timezone_id,
|
||||
"extra_http_headers": config.extra_headers,
|
||||
}
|
||||
)
|
||||
# The default useragent in the headful is always correct now in the current versions of Playwright
|
||||
if config.useragent:
|
||||
self._context_options["user_agent"] = config.useragent
|
||||
elif not config.useragent and config.headless:
|
||||
self._context_options["user_agent"] = (
|
||||
__default_chrome_useragent__ if config.real_chrome else __default_useragent__
|
||||
)
|
||||
|
||||
if not config.cdp_url:
|
||||
self._launch_options |= self._context_options
|
||||
self._context_options = {}
|
||||
flags = self._launch_options["args"]
|
||||
if config.extra_flags or extra_flags:
|
||||
flags = list(set(flags + (config.extra_flags or extra_flags)))
|
||||
|
||||
self._launch_options.update(
|
||||
{
|
||||
"args": flags,
|
||||
"headless": config.headless,
|
||||
"user_data_dir": config.user_data_dir,
|
||||
"channel": "chrome" if config.real_chrome else "chromium",
|
||||
}
|
||||
)
|
||||
|
||||
if config.additional_args:
|
||||
self._launch_options.update(config.additional_args)
|
||||
else:
|
||||
# while `context_options` is left to be used when cdp mode is enabled
|
||||
self._launch_options = dict()
|
||||
if config.additional_args:
|
||||
self._context_options.update(config.additional_args)
|
||||
|
||||
|
||||
class DynamicSessionMixin(BaseSessionMixin):
|
||||
def __validate__(self, **params):
|
||||
self._config = self.__validate_routine__(params, model=PlaywrightConfig)
|
||||
self.__generate_options__()
|
||||
|
||||
|
||||
class StealthySessionMixin(BaseSessionMixin):
|
||||
def __validate__(self, **params):
|
||||
self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig)
|
||||
self._context_options.update(
|
||||
{
|
||||
"is_mobile": False,
|
||||
"has_touch": False,
|
||||
# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
|
||||
"service_workers": "allow",
|
||||
"ignore_https_errors": True,
|
||||
"screen": {"width": 1920, "height": 1080},
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"permissions": ["geolocation", "notifications"],
|
||||
}
|
||||
)
|
||||
self.__generate_stealth_options()
|
||||
|
||||
def __generate_stealth_options(self) -> None:
|
||||
flags = tuple()
|
||||
if not self._config.cdp_url:
|
||||
flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS
|
||||
|
||||
if self._config.block_webrtc:
|
||||
flags += (
|
||||
"--webrtc-ip-handling-policy=disable_non_proxied_udp",
|
||||
"--force-webrtc-ip-handling-policy", # Ensures the policy is enforced
|
||||
)
|
||||
if not self._config.allow_webgl:
|
||||
flags += (
|
||||
"--disable-webgl",
|
||||
"--disable-webgl-image-chromium",
|
||||
"--disable-webgl2",
|
||||
)
|
||||
if self._config.hide_canvas:
|
||||
flags += ("--fingerprinting-canvas-image-data-noise",)
|
||||
|
||||
super(StealthySessionMixin, self).__generate_options__(flags)
|
||||
|
||||
@staticmethod
|
||||
def _detect_cloudflare(page_content: str) -> str | None:
|
||||
|
||||
@@ -58,88 +58,3 @@ def _set_flags(hide_canvas, disable_webgl): # pragma: no cover
|
||||
)
|
||||
|
||||
return flags
|
||||
|
||||
|
||||
@lru_cache(2, typed=True)
|
||||
def _launch_kwargs(
|
||||
headless,
|
||||
proxy: Tuple,
|
||||
locale,
|
||||
extra_headers,
|
||||
useragent,
|
||||
real_chrome,
|
||||
stealth,
|
||||
hide_canvas,
|
||||
disable_webgl,
|
||||
timezone_id,
|
||||
extra_flags: Tuple,
|
||||
) -> Tuple:
|
||||
"""Creates the arguments we will use while launching playwright's browser"""
|
||||
base_args = DEFAULT_FLAGS
|
||||
if extra_flags:
|
||||
base_args = base_args + extra_flags
|
||||
|
||||
launch_kwargs = {
|
||||
"locale": locale,
|
||||
"timezone_id": timezone_id or None,
|
||||
"headless": headless,
|
||||
"args": base_args,
|
||||
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
|
||||
"proxy": proxy or tuple(),
|
||||
"device_scale_factor": 2,
|
||||
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
|
||||
"channel": "chrome" if real_chrome else "chromium",
|
||||
"extra_http_headers": extra_headers or tuple(),
|
||||
}
|
||||
# The default useragent in the headful is always correct now in the current versions of Playwright
|
||||
if useragent:
|
||||
launch_kwargs["user_agent"] = useragent
|
||||
elif not useragent and headless:
|
||||
launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__
|
||||
|
||||
if stealth:
|
||||
stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)
|
||||
launch_kwargs.update(
|
||||
{
|
||||
"args": stealth_args,
|
||||
"chromium_sandbox": True,
|
||||
"is_mobile": False,
|
||||
"has_touch": False,
|
||||
# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
|
||||
"service_workers": "allow",
|
||||
"ignore_https_errors": True,
|
||||
"screen": {"width": 1920, "height": 1080},
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"permissions": ["geolocation", "notifications"],
|
||||
}
|
||||
)
|
||||
|
||||
return tuple(launch_kwargs.items())
|
||||
|
||||
|
||||
@lru_cache(2, typed=True)
|
||||
def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple:
|
||||
"""Creates the arguments for the browser context"""
|
||||
context_kwargs = {
|
||||
"proxy": proxy or tuple(),
|
||||
"locale": locale,
|
||||
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
|
||||
"device_scale_factor": 2,
|
||||
"extra_http_headers": extra_headers or tuple(),
|
||||
"user_agent": useragent or __default_useragent__,
|
||||
}
|
||||
if stealth:
|
||||
context_kwargs.update(
|
||||
{
|
||||
"is_mobile": False,
|
||||
"has_touch": False,
|
||||
# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
|
||||
"service_workers": "allow",
|
||||
"ignore_https_errors": True,
|
||||
"screen": {"width": 1920, "height": 1080},
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"permissions": ["geolocation", "notifications"],
|
||||
}
|
||||
)
|
||||
|
||||
return tuple(context_kwargs.items())
|
||||
|
||||
@@ -9,8 +9,6 @@ from playwright.async_api import (
|
||||
Playwright as AsyncPlaywright,
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
)
|
||||
from patchright.sync_api import sync_playwright as sync_patchright
|
||||
from patchright.async_api import async_playwright as async_patchright
|
||||
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.core._types import Unpack, TYPE_CHECKING
|
||||
@@ -21,44 +19,19 @@ from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
||||
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
|
||||
|
||||
|
||||
class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
class DynamicSession(SyncSession, DynamicSessionMixin):
|
||||
"""A Browser session manager with page pooling."""
|
||||
|
||||
__slots__ = (
|
||||
"_max_pages",
|
||||
"_headless",
|
||||
"_hide_canvas",
|
||||
"_disable_webgl",
|
||||
"_real_chrome",
|
||||
"_stealth",
|
||||
"_google_search",
|
||||
"_proxy",
|
||||
"_locale",
|
||||
"_extra_headers",
|
||||
"_useragent",
|
||||
"_timeout",
|
||||
"_cookies",
|
||||
"_disable_resources",
|
||||
"_network_idle",
|
||||
"_load_dom",
|
||||
"_wait_selector",
|
||||
"_init_script",
|
||||
"_wait_selector_state",
|
||||
"_wait",
|
||||
"playwright",
|
||||
"browser",
|
||||
"context",
|
||||
"_config",
|
||||
"_context_options",
|
||||
"_launch_options",
|
||||
"max_pages",
|
||||
"page_pool",
|
||||
"_max_wait_for_page",
|
||||
"playwright",
|
||||
"context",
|
||||
"_closed",
|
||||
"_selector_config",
|
||||
"_page_action",
|
||||
"launch_options",
|
||||
"context_options",
|
||||
"_cdp_url",
|
||||
"_headers_keys",
|
||||
"_extra_flags",
|
||||
"_additional_args",
|
||||
"_user_data_dir",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
|
||||
@@ -76,8 +49,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
|
||||
:param timezone_id: Set the timezone for the browser if wanted.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
@@ -94,27 +68,24 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
self.__validate__(**kwargs)
|
||||
super().__init__(max_pages=self._max_pages)
|
||||
super().__init__()
|
||||
|
||||
def start(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
if not self.playwright:
|
||||
sync_context = sync_patchright if self._stealth else sync_playwright
|
||||
self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
if self._cdp_url: # pragma: no cover
|
||||
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
|
||||
**self.context_options
|
||||
)
|
||||
if self._config.cdp_url: # pragma: no cover
|
||||
browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
||||
self.context = browser.new_context(**self._context_options)
|
||||
else:
|
||||
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
|
||||
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)
|
||||
|
||||
if self._init_script: # pragma: no cover
|
||||
self.context.add_init_script(path=self._init_script)
|
||||
if self._config.init_script: # pragma: no cover
|
||||
self.context.add_init_script(path=self._config.init_script)
|
||||
|
||||
if self._cookies: # pragma: no cover
|
||||
self.context.add_cookies(self._cookies)
|
||||
if self._config.cookies: # pragma: no cover
|
||||
self.context.add_cookies(self._config.cookies)
|
||||
else:
|
||||
raise RuntimeError("Session has been already started")
|
||||
|
||||
@@ -139,7 +110,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
:return: A `Response` object.
|
||||
"""
|
||||
params = _validate(kwargs, self, PlaywrightConfig)
|
||||
|
||||
if self._closed: # pragma: no cover
|
||||
raise RuntimeError("Context manager has been closed")
|
||||
|
||||
@@ -193,7 +163,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
raise e
|
||||
|
||||
|
||||
class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
|
||||
"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
|
||||
|
||||
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
|
||||
@@ -212,8 +182,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
|
||||
:param timezone_id: Set the timezone for the browser if wanted.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
@@ -230,28 +201,26 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
self.__validate__(**kwargs)
|
||||
super().__init__(max_pages=self._max_pages)
|
||||
super().__init__(max_pages=self._config.max_pages)
|
||||
|
||||
async def start(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
if not self.playwright:
|
||||
async_context = async_patchright if self._stealth else async_playwright
|
||||
self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
if self._cdp_url:
|
||||
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
|
||||
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
|
||||
if self._config.cdp_url:
|
||||
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
||||
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
|
||||
else:
|
||||
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
|
||||
**self.launch_options
|
||||
**self._launch_options
|
||||
)
|
||||
|
||||
if self._init_script: # pragma: no cover
|
||||
await self.context.add_init_script(path=self._init_script)
|
||||
if self._config.init_script: # pragma: no cover
|
||||
await self.context.add_init_script(path=self._config.init_script)
|
||||
|
||||
if self._cookies:
|
||||
await self.context.add_cookies(self._cookies) # pyright: ignore
|
||||
if self._config.cookies:
|
||||
await self.context.add_cookies(self._config.cookies) # pyright: ignore
|
||||
else:
|
||||
raise RuntimeError("Session has been already started")
|
||||
|
||||
|
||||
+113
-115
@@ -2,117 +2,102 @@ from random import randint
|
||||
from re import compile as re_compile
|
||||
|
||||
from playwright.sync_api import (
|
||||
Page,
|
||||
Locator,
|
||||
sync_playwright,
|
||||
Page,
|
||||
Playwright,
|
||||
)
|
||||
from playwright.async_api import (
|
||||
async_playwright,
|
||||
Page as async_Page,
|
||||
Locator as AsyncLocator,
|
||||
Playwright as AsyncPlaywright,
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
)
|
||||
from patchright.sync_api import sync_playwright
|
||||
from patchright.async_api import async_playwright
|
||||
|
||||
from scrapling.core.utils import log
|
||||
from ._types import CamoufoxSession, CamoufoxFetchParams
|
||||
from scrapling.core._types import Any, Unpack, TYPE_CHECKING
|
||||
from scrapling.core._types import Any, Unpack
|
||||
from ._config_tools import _compiled_stealth_scripts
|
||||
from ._types import StealthSession, StealthFetchParams
|
||||
from ._base import SyncSession, AsyncSession, StealthySessionMixin
|
||||
from ._validators import validate_fetch as _validate, CamoufoxConfig
|
||||
from ._validators import validate_fetch as _validate, StealthConfig
|
||||
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
||||
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
|
||||
|
||||
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
|
||||
|
||||
|
||||
class StealthySession(StealthySessionMixin, SyncSession):
|
||||
"""A Stealthy session manager with page pooling."""
|
||||
class StealthySession(SyncSession, StealthySessionMixin):
|
||||
"""A Stealthy Browser session manager with page pooling."""
|
||||
|
||||
__slots__ = (
|
||||
"_max_pages",
|
||||
"_headless",
|
||||
"_block_images",
|
||||
"_disable_resources",
|
||||
"_block_webrtc",
|
||||
"_allow_webgl",
|
||||
"_network_idle",
|
||||
"_load_dom",
|
||||
"_humanize",
|
||||
"_solve_cloudflare",
|
||||
"_wait",
|
||||
"_timeout",
|
||||
"_page_action",
|
||||
"_wait_selector",
|
||||
"_init_script",
|
||||
"_addons",
|
||||
"_wait_selector_state",
|
||||
"_cookies",
|
||||
"_google_search",
|
||||
"_extra_headers",
|
||||
"_proxy",
|
||||
"_os_randomize",
|
||||
"_disable_ads",
|
||||
"_geoip",
|
||||
"_selector_config",
|
||||
"_additional_args",
|
||||
"playwright",
|
||||
"browser",
|
||||
"context",
|
||||
"_config",
|
||||
"_context_options",
|
||||
"_launch_options",
|
||||
"max_pages",
|
||||
"page_pool",
|
||||
"_max_wait_for_page",
|
||||
"playwright",
|
||||
"context",
|
||||
"_closed",
|
||||
"launch_options",
|
||||
"_headers_keys",
|
||||
"_user_data_dir",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Unpack[CamoufoxSession]):
|
||||
"""A Browser session manager with page pooling
|
||||
def __init__(self, **kwargs: Unpack[StealthSession]):
|
||||
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
|
||||
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param block_images: Prevent the loading of images through Firefox preferences.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param block_webrtc: Blocks WebRTC entirely.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
|
||||
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
|
||||
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
|
||||
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
||||
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
|
||||
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
self.__validate__(**kwargs)
|
||||
super().__init__(max_pages=self._max_pages)
|
||||
super().__init__()
|
||||
|
||||
def start(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
if not self.playwright:
|
||||
self.playwright = sync_playwright().start()
|
||||
self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
|
||||
self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
if self._init_script: # pragma: no cover
|
||||
self.context.add_init_script(path=self._init_script)
|
||||
if self._config.cdp_url: # pragma: no cover
|
||||
browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
||||
self.context = browser.new_context(**self._context_options)
|
||||
else:
|
||||
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)
|
||||
|
||||
if self._cookies: # pragma: no cover
|
||||
self.context.add_cookies(self._cookies)
|
||||
for script in _compiled_stealth_scripts():
|
||||
self.context.add_init_script(script=script)
|
||||
|
||||
if self._config.init_script: # pragma: no cover
|
||||
self.context.add_init_script(path=self._config.init_script)
|
||||
|
||||
if self._config.cookies: # pragma: no cover
|
||||
self.context.add_cookies(self._config.cookies)
|
||||
else:
|
||||
raise RuntimeError("Session has been already started")
|
||||
|
||||
@@ -148,22 +133,27 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
outer_box = {}
|
||||
iframe = page.frame(url=__CF_PATTERN__)
|
||||
if iframe is not None:
|
||||
self._wait_for_page_stability(iframe, True, True)
|
||||
self._wait_for_page_stability(iframe, True, False)
|
||||
|
||||
if challenge_type != "embedded":
|
||||
while not iframe.frame_element().is_visible():
|
||||
# Double-checking that the iframe is loaded
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
outer_box: Any = iframe.frame_element().bounding_box()
|
||||
|
||||
if not iframe or not outer_box:
|
||||
if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
|
||||
log.info("Cloudflare captcha is solved")
|
||||
return
|
||||
|
||||
outer_box: Any = page.locator(box_selector).last.bounding_box()
|
||||
|
||||
# Calculate the Captcha coordinates for any viewport
|
||||
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
|
||||
|
||||
# Move the mouse to the center of the window, then press and hold the left mouse button
|
||||
page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
|
||||
page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
|
||||
self._wait_for_networkidle(page)
|
||||
if iframe is not None:
|
||||
# Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
|
||||
@@ -182,7 +172,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
log.info("Cloudflare captcha is solved")
|
||||
return
|
||||
|
||||
def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:
|
||||
def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
|
||||
"""Opens up the browser and do your request based on your chosen options.
|
||||
|
||||
:param url: The Target url.
|
||||
@@ -203,8 +193,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:return: A `Response` object.
|
||||
"""
|
||||
params = _validate(kwargs, self, CamoufoxConfig)
|
||||
|
||||
params = _validate(kwargs, self, StealthConfig)
|
||||
if self._closed: # pragma: no cover
|
||||
raise RuntimeError("Context manager has been closed")
|
||||
|
||||
@@ -233,7 +222,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
if params.page_action:
|
||||
try:
|
||||
_ = params.page_action(page_info.page)
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error executing page_action: {e}")
|
||||
|
||||
if params.wait_selector:
|
||||
@@ -242,10 +231,12 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
waiter.first.wait_for(state=params.wait_selector_state)
|
||||
# Wait again after waiting for the selector, helpful with protections like Cloudflare
|
||||
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||||
|
||||
page_info.page.wait_for_timeout(params.wait)
|
||||
|
||||
# Create response object
|
||||
response = ResponseFactory.from_playwright_response(
|
||||
page_info.page, first_response, final_response[0], params.selector_config
|
||||
)
|
||||
@@ -256,72 +247,79 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
except Exception as e:
|
||||
page_info.mark_error()
|
||||
raise e
|
||||
|
||||
|
||||
class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
"""A Stealthy session manager with page pooling."""
|
||||
class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
||||
"""An async Stealthy Browser session manager with page pooling."""
|
||||
|
||||
def __init__(self, **kwargs: Unpack[CamoufoxSession]):
|
||||
"""A Browser session manager with page pooling
|
||||
def __init__(self, **kwargs: Unpack[StealthSession]):
|
||||
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
|
||||
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param block_images: Prevent the loading of images through Firefox preferences.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param block_webrtc: Blocks WebRTC entirely.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
|
||||
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
|
||||
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
|
||||
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
||||
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
|
||||
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
|
||||
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
self.__validate__(**kwargs)
|
||||
super().__init__(max_pages=self._max_pages)
|
||||
super().__init__(max_pages=self._config.max_pages)
|
||||
|
||||
async def start(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
if not self.playwright:
|
||||
self.playwright: AsyncPlaywright = await async_playwright().start()
|
||||
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
|
||||
**self.launch_options
|
||||
)
|
||||
self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
if self._init_script: # pragma: no cover
|
||||
await self.context.add_init_script(path=self._init_script)
|
||||
if self._config.cdp_url:
|
||||
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
||||
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
|
||||
else:
|
||||
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
|
||||
**self._launch_options
|
||||
)
|
||||
|
||||
if self._cookies:
|
||||
await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]
|
||||
for script in _compiled_stealth_scripts():
|
||||
await self.context.add_init_script(script=script)
|
||||
|
||||
if self._config.init_script: # pragma: no cover
|
||||
await self.context.add_init_script(path=self._config.init_script)
|
||||
|
||||
if self._config.cookies:
|
||||
await self.context.add_cookies(self._config.cookies) # pyright: ignore
|
||||
else:
|
||||
raise RuntimeError("Session has been already started")
|
||||
|
||||
async def _cloudflare_solver(self, page: async_Page): # pragma: no cover
|
||||
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
|
||||
async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover
|
||||
"""Solve the cloudflare challenge displayed on the playwright page passed
|
||||
|
||||
:param page: The async targeted page
|
||||
:param page: The targeted page
|
||||
:return:
|
||||
"""
|
||||
await self._wait_for_networkidle(page, timeout=5000)
|
||||
@@ -331,7 +329,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
return
|
||||
else:
|
||||
log.info(f'The turnstile version discovered is "{challenge_type}"')
|
||||
if challenge_type == "non-interactive": # pragma: no cover
|
||||
if challenge_type == "non-interactive":
|
||||
while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
|
||||
log.info("Waiting for Cloudflare wait page to disappear.")
|
||||
await page.wait_for_timeout(1000)
|
||||
@@ -350,22 +348,27 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
outer_box = {}
|
||||
iframe = page.frame(url=__CF_PATTERN__)
|
||||
if iframe is not None:
|
||||
await self._wait_for_page_stability(iframe, True, True)
|
||||
await self._wait_for_page_stability(iframe, True, False)
|
||||
|
||||
if challenge_type != "embedded":
|
||||
while not await (await iframe.frame_element()).is_visible():
|
||||
# Double-checking that the iframe is loaded
|
||||
await page.wait_for_timeout(500)
|
||||
outer_box: Any = await (await iframe.frame_element()).bounding_box()
|
||||
|
||||
outer_box: Any = (await iframe.frame_element()).bounding_box()
|
||||
|
||||
if not iframe or not outer_box:
|
||||
if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
|
||||
log.info("Cloudflare captcha is solved")
|
||||
return
|
||||
|
||||
outer_box: Any = await page.locator(box_selector).last.bounding_box()
|
||||
|
||||
# Calculate the Captcha coordinates for any viewport
|
||||
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
|
||||
|
||||
# Move the mouse to the center of the window, then press and hold the left mouse button
|
||||
await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
|
||||
await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
|
||||
await self._wait_for_networkidle(page)
|
||||
if iframe is not None:
|
||||
# Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
|
||||
@@ -377,14 +380,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
await page.wait_for_timeout(100)
|
||||
attempts += 1
|
||||
if challenge_type != "embedded":
|
||||
await page.locator(box_selector).wait_for(state="detached")
|
||||
await page.locator(box_selector).last.wait_for(state="detached")
|
||||
await page.locator(".zone-name-title").wait_for(state="hidden")
|
||||
await self._wait_for_page_stability(page, True, False)
|
||||
|
||||
log.info("Cloudflare captcha is solved")
|
||||
return
|
||||
|
||||
async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:
|
||||
async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
|
||||
"""Opens up the browser and do your request based on your chosen options.
|
||||
|
||||
:param url: The Target url.
|
||||
@@ -405,7 +408,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:return: A `Response` object.
|
||||
"""
|
||||
params = _validate(kwargs, self, CamoufoxConfig)
|
||||
params = _validate(kwargs, self, StealthConfig)
|
||||
|
||||
if self._closed: # pragma: no cover
|
||||
raise RuntimeError("Context manager has been closed")
|
||||
@@ -418,10 +421,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
final_response = [None]
|
||||
handle_response = self._create_response_handler(page_info, final_response)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
if not isinstance(page_info.page, async_Page):
|
||||
raise TypeError
|
||||
|
||||
try:
|
||||
# Navigate to URL and wait for a specified state
|
||||
page_info.page.on("response", handle_response)
|
||||
@@ -461,9 +460,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
# Close the page to free up resources
|
||||
await page_info.page.close()
|
||||
self.page_pool.pages.remove(page_info)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
page_info.mark_error()
|
||||
raise e
|
||||
@@ -53,7 +53,7 @@ if TYPE_CHECKING: # pragma: no cover
|
||||
json: Optional[Dict | List]
|
||||
|
||||
# Types for browser session
|
||||
class BrowserSession(TypedDict, total=False):
|
||||
class PlaywrightSession(TypedDict, total=False):
|
||||
max_pages: int
|
||||
headless: bool
|
||||
disable_resources: bool
|
||||
@@ -64,6 +64,7 @@ if TYPE_CHECKING: # pragma: no cover
|
||||
cookies: Optional[Iterable[Dict]]
|
||||
google_search: bool
|
||||
wait: int | float
|
||||
timezone_id: str | None
|
||||
page_action: Optional[Callable]
|
||||
proxy: Optional[str | Dict[str, str] | Tuple]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
@@ -72,42 +73,32 @@ if TYPE_CHECKING: # pragma: no cover
|
||||
user_data_dir: str
|
||||
selector_config: Optional[Dict]
|
||||
additional_args: Optional[Dict]
|
||||
|
||||
class PlaywrightSession(BrowserSession, total=False):
|
||||
cdp_url: Optional[str]
|
||||
hide_canvas: bool
|
||||
disable_webgl: bool
|
||||
locale: Optional[str]
|
||||
real_chrome: bool
|
||||
stealth: bool
|
||||
locale: str
|
||||
cdp_url: Optional[str]
|
||||
useragent: Optional[str]
|
||||
extra_flags: Optional[List[str]]
|
||||
|
||||
class PlaywrightFetchParams(TypedDict, total=False):
|
||||
load_dom: bool
|
||||
wait: int | float
|
||||
network_idle: bool
|
||||
google_search: bool
|
||||
timeout: int | float
|
||||
wait: int | float
|
||||
page_action: Optional[Callable]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
disable_resources: bool
|
||||
wait_selector: Optional[str]
|
||||
wait_selector_state: SelectorWaitStates
|
||||
network_idle: bool
|
||||
load_dom: bool
|
||||
page_action: Optional[Callable]
|
||||
selector_config: Optional[Dict]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
wait_selector_state: SelectorWaitStates
|
||||
|
||||
class CamoufoxSession(BrowserSession, total=False):
|
||||
block_images: bool
|
||||
block_webrtc: bool
|
||||
class StealthSession(PlaywrightSession, total=False):
|
||||
allow_webgl: bool
|
||||
humanize: bool | float
|
||||
hide_canvas: bool
|
||||
block_webrtc: bool
|
||||
solve_cloudflare: bool
|
||||
addons: Optional[List[str]]
|
||||
os_randomize: bool
|
||||
disable_ads: bool
|
||||
geoip: bool
|
||||
|
||||
class CamoufoxFetchParams(PlaywrightFetchParams, total=False):
|
||||
class StealthFetchParams(PlaywrightFetchParams, total=False):
|
||||
solve_cloudflare: bool
|
||||
|
||||
else: # pragma: no cover
|
||||
@@ -116,5 +107,5 @@ else: # pragma: no cover
|
||||
DataRequestParams = TypedDict
|
||||
PlaywrightSession = TypedDict
|
||||
PlaywrightFetchParams = TypedDict
|
||||
CamoufoxSession = TypedDict
|
||||
CamoufoxFetchParams = TypedDict
|
||||
StealthSession = TypedDict
|
||||
StealthFetchParams = TypedDict
|
||||
|
||||
@@ -14,11 +14,13 @@ from scrapling.core._types import (
|
||||
Optional,
|
||||
Callable,
|
||||
Iterable,
|
||||
SelectorWaitStates,
|
||||
Sequence,
|
||||
overload,
|
||||
SetCookieParam,
|
||||
SelectorWaitStates,
|
||||
)
|
||||
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
|
||||
from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams
|
||||
from scrapling.engines._browsers._types import PlaywrightFetchParams, StealthFetchParams
|
||||
|
||||
|
||||
# Custom validators for msgspec
|
||||
@@ -68,26 +70,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
cdp_url: Optional[str] = None
|
||||
headless: bool = True
|
||||
google_search: bool = True
|
||||
hide_canvas: bool = False
|
||||
disable_webgl: bool = False
|
||||
# hide_canvas: bool = False
|
||||
# disable_webgl: bool = False
|
||||
real_chrome: bool = False
|
||||
stealth: bool = False
|
||||
# stealth: bool = False
|
||||
wait: Seconds = 0
|
||||
page_action: Optional[Callable] = None
|
||||
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
|
||||
locale: str = "en-US"
|
||||
locale: str | None = None
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
useragent: Optional[str] = None
|
||||
timeout: Seconds = 30000
|
||||
init_script: Optional[str] = None
|
||||
disable_resources: bool = False
|
||||
wait_selector: Optional[str] = None
|
||||
cookies: Optional[Iterable[Dict]] = None
|
||||
cookies: Sequence[SetCookieParam] | None = []
|
||||
network_idle: bool = False
|
||||
load_dom: bool = True
|
||||
wait_selector_state: SelectorWaitStates = "attached"
|
||||
user_data_dir: str = ""
|
||||
timezone_id: str = ""
|
||||
timezone_id: str | None = ""
|
||||
extra_flags: Optional[List[str]] = None
|
||||
selector_config: Optional[Dict] = {}
|
||||
additional_args: Optional[Dict] = {}
|
||||
@@ -118,64 +120,18 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
raise ValueError(validation_msg)
|
||||
|
||||
|
||||
class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
"""Configuration struct for validation"""
|
||||
|
||||
max_pages: PagesCount = 1
|
||||
headless: bool = True # noqa: F821
|
||||
block_images: bool = False
|
||||
disable_resources: bool = False
|
||||
block_webrtc: bool = False
|
||||
class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
|
||||
allow_webgl: bool = True
|
||||
network_idle: bool = False
|
||||
load_dom: bool = True
|
||||
humanize: bool | float = True
|
||||
hide_canvas: bool = False
|
||||
block_webrtc: bool = False
|
||||
solve_cloudflare: bool = False
|
||||
wait: Seconds = 0
|
||||
timeout: Seconds = 30000
|
||||
init_script: Optional[str] = None
|
||||
page_action: Optional[Callable] = None
|
||||
wait_selector: Optional[str] = None
|
||||
addons: Optional[List[str]] = None
|
||||
wait_selector_state: SelectorWaitStates = "attached"
|
||||
cookies: Optional[Iterable[Dict]] = None
|
||||
google_search: bool = True
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
|
||||
os_randomize: bool = False
|
||||
disable_ads: bool = False
|
||||
geoip: bool = False
|
||||
user_data_dir: str = ""
|
||||
selector_config: Optional[Dict] = {}
|
||||
additional_args: Optional[Dict] = {}
|
||||
|
||||
def __post_init__(self):
|
||||
"""Custom validation after msgspec validation"""
|
||||
if self.page_action and not callable(self.page_action):
|
||||
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
|
||||
if self.proxy:
|
||||
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
|
||||
|
||||
if self.addons:
|
||||
for addon in self.addons:
|
||||
_validate_addon_path(addon)
|
||||
else:
|
||||
self.addons = []
|
||||
|
||||
if self.init_script is not None:
|
||||
validation_msg = _is_invalid_file_path(self.init_script)
|
||||
if validation_msg:
|
||||
raise ValueError(validation_msg)
|
||||
|
||||
if not self.cookies:
|
||||
self.cookies = []
|
||||
super(StealthConfig, self).__post_init__()
|
||||
# Cloudflare timeout adjustment
|
||||
if self.solve_cloudflare and self.timeout < 60_000:
|
||||
self.timeout = 60_000
|
||||
if not self.selector_config:
|
||||
self.selector_config = {}
|
||||
if not self.additional_args:
|
||||
self.additional_args = {}
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -197,9 +153,9 @@ class _fetch_params:
|
||||
|
||||
|
||||
def validate_fetch(
|
||||
method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams,
|
||||
method_kwargs: Dict | PlaywrightFetchParams | StealthFetchParams,
|
||||
session: Any,
|
||||
model: type[PlaywrightConfig] | type[CamoufoxConfig],
|
||||
model: type[PlaywrightConfig] | type[StealthConfig],
|
||||
) -> _fetch_params: # pragma: no cover
|
||||
result = {}
|
||||
overrides = {}
|
||||
@@ -210,21 +166,20 @@ def validate_fetch(
|
||||
for key in fetch_param_fields:
|
||||
if key in method_kwargs:
|
||||
overrides[key] = method_kwargs[key]
|
||||
else:
|
||||
# Check for underscore-prefixed attribute (private)
|
||||
attr_name = f"_{key}"
|
||||
if hasattr(session, attr_name):
|
||||
result[key] = getattr(session, attr_name)
|
||||
elif hasattr(session, "_config") and hasattr(session._config, key):
|
||||
result[key] = getattr(session._config, key)
|
||||
|
||||
if overrides:
|
||||
validated_config = validate(overrides, model)
|
||||
# Extract only the fields that _fetch_params needs from validated_config
|
||||
# Extract ONLY the fields that were actually overridden (not all fields)
|
||||
# This prevents validated defaults from overwriting session config values
|
||||
validated_dict = {
|
||||
f.name: getattr(validated_config, f.name)
|
||||
for f in fields(_fetch_params)
|
||||
if hasattr(validated_config, f.name)
|
||||
field: getattr(validated_config, field) for field in overrides.keys() if hasattr(validated_config, field)
|
||||
}
|
||||
validated_dict.setdefault("solve_cloudflare", False)
|
||||
|
||||
# Preserve solve_cloudflare if the user explicitly provided it, even if the model doesn't have it
|
||||
if "solve_cloudflare" in overrides:
|
||||
validated_dict["solve_cloudflare"] = overrides["solve_cloudflare"]
|
||||
|
||||
# Start with session defaults, then overwrite with validated overrides
|
||||
result.update(validated_dict)
|
||||
@@ -238,7 +193,7 @@ def validate_fetch(
|
||||
# Cache default values for each model to reduce validation overhead
|
||||
models_default_values = {}
|
||||
|
||||
for _model in (CamoufoxConfig, PlaywrightConfig):
|
||||
for _model in (StealthConfig, PlaywrightConfig):
|
||||
_defaults = {}
|
||||
if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"):
|
||||
for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore
|
||||
@@ -256,14 +211,14 @@ def _filter_defaults(params: Dict, model: str) -> Dict:
|
||||
|
||||
|
||||
@overload
|
||||
def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
|
||||
def validate(params: Dict, model: type[StealthConfig]) -> StealthConfig: ...
|
||||
|
||||
|
||||
@overload
|
||||
def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
|
||||
def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
|
||||
|
||||
|
||||
def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig:
|
||||
def validate(params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]) -> PlaywrightConfig | StealthConfig:
|
||||
try:
|
||||
# Filter out params with the default values (no need to validate them) to speed up validation
|
||||
filtered = _filter_defaults(params, model.__name__)
|
||||
|
||||
Reference in New Issue
Block a user