freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(browser fetchers): Make all the type hints dynamic + Faster validation

Browse Source 
Made the code shorter by an additional ~200 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython.
This commit is contained in:
Karim shoair
2025-11-25 20:18:36 +02:00
parent 9e85311ff6
commit f0b01fc253
7 changed files with 364 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+81 -259
View File
@@ -13,92 +13,55 @@ from patchright.sync_api import sync_playwright as sync_patchright
from patchright.async_api import async_playwright as async_patchright
from scrapling.core.utils import log
from scrapling.core._types import Unpack, TYPE_CHECKING
from ._types import PlaywrightSession, PlaywrightFetchParams
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate_fetch as _validate, PlaywrightConfig
from scrapling.core._types import (
Any,
Dict,
List,
Optional,
Callable,
TYPE_CHECKING,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.convertor import (
Response,
ResponseFactory,
)
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
_UNSET: Any = object()
class DynamicSession(DynamicSessionMixin, SyncSession):
"""A Browser session manager with page pooling."""
__slots__ = (
"max_pages",
"headless",
"hide_canvas",
"disable_webgl",
"real_chrome",
"stealth",
"google_search",
"proxy",
"locale",
"extra_headers",
"useragent",
"timeout",
"cookies",
"disable_resources",
"network_idle",
"load_dom",
"wait_selector",
"init_script",
"wait_selector_state",
"wait",
"_max_pages",
"_headless",
"_hide_canvas",
"_disable_webgl",
"_real_chrome",
"_stealth",
"_google_search",
"_proxy",
"_locale",
"_extra_headers",
"_useragent",
"_timeout",
"_cookies",
"_disable_resources",
"_network_idle",
"_load_dom",
"_wait_selector",
"_init_script",
"_wait_selector_state",
"_wait",
"playwright",
"browser",
"context",
"page_pool",
"_closed",
"selector_config",
"page_action",
"_selector_config",
"_page_action",
"launch_options",
"context_options",
"cdp_url",
"_cdp_url",
"_headers_keys",
"_extra_flags",
"_additional_args",
"_user_data_dir",
)
def __init__(
self,
__max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
extra_flags: Optional[List[str]] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -129,105 +92,49 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
load_dom=load_dom,
headless=headless,
useragent=useragent,
max_pages=__max_pages,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
extra_flags=extra_flags,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
def __create__(self):
"""Create a browser for this instance and context."""
sync_context = sync_patchright if self.stealth else sync_playwright
sync_context = sync_patchright if self._stealth else sync_playwright
self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
if self._cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
**self.context_options
)
else:
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies)
def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
PlaywrightConfig,
_UNSET,
)
params = _validate(kwargs, self, PlaywrightConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -285,35 +192,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
def __init__(
self,
max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
extra_flags: Optional[List[str]] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -345,107 +224,50 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
load_dom=load_dom,
headless=headless,
useragent=useragent,
max_pages=max_pages,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
extra_flags=extra_flags,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
async def __create__(self):
"""Create a browser for this instance and context."""
async_context = async_patchright if self.stealth else async_playwright
async_context = async_patchright if self._stealth else async_playwright
self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
if self._cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self.launch_options
)
if self.init_script: # pragma: no cover
await self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script)
if self.cookies:
await self.context.add_cookies(self.cookies) # pyright: ignore
if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore
async def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
PlaywrightConfig,
_UNSET,
)
params = _validate(kwargs, self, PlaywrightConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")