freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: Improving all browser-based fetches by ~15%

Browse Source
This commit is contained in:
Karim shoair
2025-09-20 06:20:23 +03:00
parent 970ad40404
commit 82b09620b2
4 changed files with 195 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
-5
View File
@@ -60,11 +60,6 @@ class SyncSession:
return self.page_pool.add_page(page)
@staticmethod
def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:
"""Get value with request-level priority over session-level"""
return request_value if request_value is not sentinel_value else session_value
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
scrapling/engines/_browsers/_camoufox.py
+33 -34
View File
@@ -16,7 +16,7 @@ from playwright.async_api import (
)
from playwright._impl._errors import Error as PlaywrightError
from ._validators import validate, CamoufoxConfig
from ._validators import validate_fetch as _validate
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
@@ -297,23 +297,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
CamoufoxConfig,
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
_UNSET,
)
if self._closed: # pragma: no cover
@@ -617,22 +616,22 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
CamoufoxConfig,
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
_UNSET,
)
if self._closed: # pragma: no cover
scrapling/engines/_browsers/_controllers.py
+31 -33
View File
@@ -16,7 +16,7 @@ from patchright.async_api import async_playwright as async_patchright
from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate, PlaywrightConfig
from ._validators import validate_fetch as _validate
from scrapling.core._types import (
Dict,
List,
@@ -224,22 +224,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
PlaywrightConfig,
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
_UNSET,
)
if self._closed: # pragma: no cover
@@ -471,22 +470,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
PlaywrightConfig,
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
_UNSET,
)
if self._closed: # pragma: no cover
scrapling/engines/_browsers/_validators.py
+131 -66
View File
@@ -1,21 +1,69 @@
from msgspec import Struct, convert, ValidationError
from urllib.parse import urlparse
from pathlib import Path
from typing import Annotated
from dataclasses import dataclass
from urllib.parse import urlparse
from msgspec import Struct, Meta, convert, ValidationError
from scrapling.core._types import (
Optional,
Dict,
Callable,
List,
Tuple,
Optional,
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
# Custom validators for msgspec
def _validate_file_path(value: str):
"""Fast file path validation"""
path = Path(value)
if not path.exists():
raise ValueError(f"Init script path not found: {value}")
if not path.is_file():
raise ValueError(f"Init script is not a file: {value}")
if not path.is_absolute():
raise ValueError(f"Init script is not a absolute path: {value}")
def _validate_addon_path(value: str):
"""Fast addon path validation"""
path = Path(value)
if not path.exists():
raise FileNotFoundError(f"Addon path not found: {value}")
if not path.is_dir():
raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")
def _validate_cdp_url(cdp_url: str):
"""Fast CDP URL validation"""
try:
# Check the scheme
if not cdp_url.startswith(("ws://", "wss://")):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not urlparse(cdp_url).netloc:
raise ValueError("Invalid hostname for the CDP URL")
except AttributeError as e:
raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
except Exception as e:
raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
# Type aliases for cleaner annotations
PagesCount = Annotated[int, Meta(ge=1, le=50)]
Seconds = Annotated[int, float, Meta(ge=0)]
class PlaywrightConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
max_pages: PagesCount = 1
cdp_url: Optional[str] = None
headless: bool = True
google_search: bool = True
@@ -23,13 +71,13 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
disable_webgl: bool = False
real_chrome: bool = False
stealth: bool = False
wait: int | float = 0
wait: Seconds = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: int | float = 30000
timeout: Seconds = 30000
init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
@@ -41,52 +89,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
def __post_init__(self):
"""Custom validation after msgspec validation"""
if self.max_pages < 1 or self.max_pages > 50:
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
self.__validate_cdp(self.cdp_url)
_validate_cdp_url(self.cdp_url)
if not self.cookies:
self.cookies = []
if not self.selector_config:
self.selector_config = {}
if self.init_script is not None:
script_path = Path(self.init_script)
if not script_path.exists():
raise ValueError("Init script path not found")
elif not script_path.is_file():
raise ValueError("Init script is not a file")
elif not script_path.is_absolute():
raise ValueError("Init script is not a absolute path")
@staticmethod
def __validate_cdp(cdp_url):
try:
# Check the scheme
if not cdp_url.startswith(("ws://", "wss://")):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not urlparse(cdp_url).netloc:
raise ValueError("Invalid hostname for the CDP URL")
except AttributeError as e:
raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
except Exception as e:
raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
_validate_file_path(self.init_script)
class CamoufoxConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
max_pages: PagesCount = 1
headless: bool = True # noqa: F821
block_images: bool = False
disable_resources: bool = False
@@ -96,8 +118,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
load_dom: bool = True
humanize: bool | float = True
solve_cloudflare: bool = False
wait: int | float = 0
timeout: int | float = 30000
wait: Seconds = 0
timeout: Seconds = 30000
init_script: Optional[str] = None
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
@@ -115,38 +137,23 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
def __post_init__(self):
"""Custom validation after msgspec validation"""
if self.max_pages < 1 or self.max_pages > 50:
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if not self.addons:
self.addons = []
else:
if self.addons and isinstance(self.addons, list):
for addon in self.addons:
addon_path = Path(addon)
if not addon_path.exists():
raise FileNotFoundError(f"Addon's path not found: {addon}")
elif not addon_path.is_dir():
raise ValueError(
f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
)
_validate_addon_path(addon)
else:
self.addons = []
if self.init_script is not None:
script_path = Path(self.init_script)
if not script_path.exists():
raise ValueError("Init script path not found")
elif not script_path.is_file():
raise ValueError("Init script is not a file")
elif not script_path.is_absolute():
raise ValueError("Init script is not a absolute path")
_validate_file_path(self.init_script)
if not self.cookies:
self.cookies = []
# Cloudflare timeout adjustment
if self.solve_cloudflare and self.timeout < 60_000:
self.timeout = 60_000
if not self.selector_config:
@@ -155,10 +162,68 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.additional_args = {}
def validate(params, model):
try:
config = convert(params, model)
except ValidationError as e:
raise TypeError(f"Invalid argument type: {e}")
# Code parts to validate `fetch` in the least possible numbers of lines overall
class FetchConfig(Struct, kw_only=True):
"""Configuration struct for `fetch` calls validation"""
return config
google_search: bool = True
timeout: Seconds = 30000
wait: Seconds = 0
page_action: Optional[Callable] = None
extra_headers: Optional[Dict[str, str]] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
wait_selector_state: SelectorWaitStates = "attached"
network_idle: bool = False
load_dom: bool = True
solve_cloudflare: bool = False
selector_config: Optional[Dict] = {}
def to_dict(self):
return {f: getattr(self, f) for f in self.__struct_fields__}
@dataclass
class _fetch_params:
"""A dataclass of all parameters used by `fetch` calls"""
google_search: bool
timeout: Seconds
wait: Seconds
page_action: Optional[Callable]
extra_headers: Optional[Dict[str, str]]
disable_resources: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
network_idle: bool
load_dom: bool
solve_cloudflare: bool
selector_config: Optional[Dict]
def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
result = {}
overrides = {}
for arg, request_value, session_value in params:
if request_value is not sentinel:
overrides[arg] = request_value
else:
result[arg] = session_value
if overrides:
overrides = validate(overrides, FetchConfig).to_dict()
overrides.update(result)
return _fetch_params(**overrides)
if not result.get("solve_cloudflare"):
result["solve_cloudflare"] = False
return _fetch_params(**result)
def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
try:
return convert(params, model)
except ValidationError as e:
raise TypeError(f"Invalid argument type: {e}") from e