feat(spiders/fetchers): Adding proxy rotation logic and change retry logic
- User passes a single proxy to the browser session, and it will keep using the same context. Pass a proxy manager that automatically refreshes the IP with each proxy, and you get speed but sacrifice some stealth. - User imports the proxy rotator class and passes proxies to it and a rotation strategy (Round robin by default). Then pass the instance to the session class, which will create a context and a tab for each proxy returned by the rotator. This way, you sacrifice speed a bit, but you get maximum stealth since each context is created with the proxy that will be used. - Normal requests use what you pass without issues, of course. - All errors are retried now, and proxies are rotated on retries automatically.
This commit is contained in:
@@ -18,6 +18,7 @@ from scrapling.core._types import (
|
||||
SetCookieParam,
|
||||
SelectorWaitStates,
|
||||
)
|
||||
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
|
||||
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
|
||||
from scrapling.engines._browsers._types import PlaywrightFetchParams, StealthFetchParams
|
||||
|
||||
@@ -70,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
timezone_id: str | None = ""
|
||||
page_action: Optional[Callable] = None
|
||||
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
|
||||
proxy_rotator: Optional[ProxyRotator] = None
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
timeout: Seconds = 30000
|
||||
init_script: Optional[str] = None
|
||||
@@ -88,6 +90,11 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
"""Custom validation after msgspec validation"""
|
||||
if self.page_action and not callable(self.page_action):
|
||||
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
|
||||
if self.proxy and self.proxy_rotator:
|
||||
raise ValueError(
|
||||
"Cannot use 'proxy_rotator' together with 'proxy'. "
|
||||
"Use either a static proxy or proxy rotation, not both."
|
||||
)
|
||||
if self.proxy:
|
||||
self.proxy = construct_proxy_dict(self.proxy)
|
||||
if self.cdp_url:
|
||||
|
||||
Reference in New Issue
Block a user