refactor: huge change, many features/class got a better naming
- `Adaptor` became `Selector` - `Adaptors` became `Selectors` - `auto_match` argument/feature became `adaptive` - `adaptor_arguments` argument became `selector_config` - `automatch_domain` argument became `adaptive_domain` - `additional_arguments` argument became `additional_args` - `storage_adaptors` file became just `storage`
This commit is contained in:
Karim shoair
@@ -70,8 +70,8 @@ class StealthySession:
|
||||
"os_randomize",
|
||||
"disable_ads",
|
||||
"geoip",
|
||||
"adaptor_arguments",
|
||||
"additional_arguments",
|
||||
"selector_config",
|
||||
"additional_args",
|
||||
"playwright",
|
||||
"browser",
|
||||
"context",
|
||||
@@ -105,8 +105,8 @@ class StealthySession:
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
adaptor_arguments: Optional[Dict] = None,
|
||||
additional_arguments: Optional[Dict] = None,
|
||||
selector_config: Optional[Dict] = None,
|
||||
additional_args: Optional[Dict] = None,
|
||||
):
|
||||
"""A Browser session manager with page pooling
|
||||
|
||||
@@ -136,8 +136,8 @@ class StealthySession:
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
|
||||
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
|
||||
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
|
||||
params = {
|
||||
@@ -163,8 +163,8 @@ class StealthySession:
|
||||
"os_randomize": os_randomize,
|
||||
"disable_ads": disable_ads,
|
||||
"geoip": geoip,
|
||||
"adaptor_arguments": adaptor_arguments,
|
||||
"additional_arguments": additional_arguments,
|
||||
"selector_config": selector_config,
|
||||
"additional_args": additional_args,
|
||||
}
|
||||
config = validate(params, CamoufoxConfig)
|
||||
|
||||
@@ -190,14 +190,14 @@ class StealthySession:
|
||||
self.os_randomize = config.os_randomize
|
||||
self.disable_ads = config.disable_ads
|
||||
self.geoip = config.geoip
|
||||
self.adaptor_arguments = config.adaptor_arguments
|
||||
self.additional_arguments = config.additional_arguments
|
||||
self.selector_config = config.selector_config
|
||||
self.additional_args = config.additional_args
|
||||
|
||||
self.playwright: Optional[Playwright] = None
|
||||
self.context: Optional[BrowserContext] = None
|
||||
self.page_pool = PagePool(self.max_pages)
|
||||
self._closed = False
|
||||
self.adaptor_arguments = config.adaptor_arguments
|
||||
self.selector_config = config.selector_config
|
||||
self.page_action = config.page_action
|
||||
self._headers_keys = (
|
||||
set(map(str.lower, self.extra_headers.keys()))
|
||||
@@ -223,7 +223,7 @@ class StealthySession:
|
||||
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
|
||||
"os": None if self.os_randomize else get_os_name(),
|
||||
"user_data_dir": "",
|
||||
**self.additional_arguments,
|
||||
**self.additional_args,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -433,7 +433,7 @@ class StealthySession:
|
||||
|
||||
page_info.page.wait_for_timeout(self.wait)
|
||||
response = ResponseFactory.from_playwright_response(
|
||||
page_info.page, first_response, final_response, self.adaptor_arguments
|
||||
page_info.page, first_response, final_response, self.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as ready for next use
|
||||
@@ -482,8 +482,8 @@ class AsyncStealthySession(StealthySession):
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
adaptor_arguments: Optional[Dict] = None,
|
||||
additional_arguments: Optional[Dict] = None,
|
||||
selector_config: Optional[Dict] = None,
|
||||
additional_args: Optional[Dict] = None,
|
||||
):
|
||||
"""A Browser session manager with page pooling
|
||||
|
||||
@@ -513,8 +513,8 @@ class AsyncStealthySession(StealthySession):
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
|
||||
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
|
||||
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
super().__init__(
|
||||
max_pages,
|
||||
@@ -539,8 +539,8 @@ class AsyncStealthySession(StealthySession):
|
||||
os_randomize,
|
||||
disable_ads,
|
||||
geoip,
|
||||
adaptor_arguments,
|
||||
additional_arguments,
|
||||
selector_config,
|
||||
additional_args,
|
||||
)
|
||||
self.playwright: Optional[AsyncPlaywright] = None
|
||||
self.context: Optional[AsyncBrowserContext] = None
|
||||
@@ -731,7 +731,7 @@ class AsyncStealthySession(StealthySession):
|
||||
|
||||
# Create response object
|
||||
response = await ResponseFactory.from_async_playwright_response(
|
||||
page_info.page, first_response, final_response, self.adaptor_arguments
|
||||
page_info.page, first_response, final_response, self.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as ready for next use
|
||||
|
||||
@@ -70,7 +70,7 @@ class DynamicSession:
|
||||
"context",
|
||||
"page_pool",
|
||||
"_closed",
|
||||
"adaptor_arguments",
|
||||
"selector_config",
|
||||
"page_action",
|
||||
"launch_options",
|
||||
"context_options",
|
||||
@@ -100,7 +100,7 @@ class DynamicSession:
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
network_idle: bool = False,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
adaptor_arguments: Optional[Dict] = None,
|
||||
selector_config: Optional[Dict] = None,
|
||||
):
|
||||
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
|
||||
|
||||
@@ -125,7 +125,7 @@ class DynamicSession:
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
"""
|
||||
|
||||
params = {
|
||||
@@ -143,7 +143,7 @@ class DynamicSession:
|
||||
"extra_headers": extra_headers,
|
||||
"useragent": useragent,
|
||||
"timeout": timeout,
|
||||
"adaptor_arguments": adaptor_arguments,
|
||||
"selector_config": selector_config,
|
||||
"disable_resources": disable_resources,
|
||||
"wait_selector": wait_selector,
|
||||
"cookies": cookies,
|
||||
@@ -177,7 +177,7 @@ class DynamicSession:
|
||||
self.context: Optional[BrowserContext] = None
|
||||
self.page_pool = PagePool(self.max_pages)
|
||||
self._closed = False
|
||||
self.adaptor_arguments = config.adaptor_arguments
|
||||
self.selector_config = config.selector_config
|
||||
self.page_action = config.page_action
|
||||
self._headers_keys = (
|
||||
set(map(str.lower, self.extra_headers.keys()))
|
||||
@@ -370,7 +370,7 @@ class DynamicSession:
|
||||
|
||||
# Create response object
|
||||
response = ResponseFactory.from_playwright_response(
|
||||
page_info.page, first_response, final_response, self.adaptor_arguments
|
||||
page_info.page, first_response, final_response, self.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as ready for next use
|
||||
@@ -417,7 +417,7 @@ class AsyncDynamicSession(DynamicSession):
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
network_idle: bool = False,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
adaptor_arguments: Optional[Dict] = None,
|
||||
selector_config: Optional[Dict] = None,
|
||||
):
|
||||
"""A Browser session manager with page pooling
|
||||
|
||||
@@ -443,7 +443,7 @@ class AsyncDynamicSession(DynamicSession):
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
|
||||
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
"""
|
||||
|
||||
super().__init__(
|
||||
@@ -467,7 +467,7 @@ class AsyncDynamicSession(DynamicSession):
|
||||
cookies,
|
||||
network_idle,
|
||||
wait_selector_state,
|
||||
adaptor_arguments,
|
||||
selector_config,
|
||||
)
|
||||
|
||||
self.playwright: Optional[AsyncPlaywright] = None
|
||||
@@ -623,7 +623,7 @@ class AsyncDynamicSession(DynamicSession):
|
||||
|
||||
# Create response object
|
||||
response = await ResponseFactory.from_async_playwright_response(
|
||||
page_info.page, first_response, final_response, self.adaptor_arguments
|
||||
page_info.page, first_response, final_response, self.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as ready for next use
|
||||
|
||||
@@ -39,7 +39,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
|
||||
cookies: Optional[List[Dict]] = None
|
||||
network_idle: bool = False
|
||||
wait_selector_state: SelectorWaitStates = "attached"
|
||||
adaptor_arguments: Optional[Dict] = None
|
||||
selector_config: Optional[Dict] = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Custom validation after msgspec validation"""
|
||||
@@ -57,8 +57,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
|
||||
self.__validate_cdp(self.cdp_url)
|
||||
if not self.cookies:
|
||||
self.cookies = []
|
||||
if not self.adaptor_arguments:
|
||||
self.adaptor_arguments = {}
|
||||
if not self.selector_config:
|
||||
self.selector_config = {}
|
||||
|
||||
@staticmethod
|
||||
def __validate_cdp(cdp_url):
|
||||
@@ -105,8 +105,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
|
||||
os_randomize: bool = False
|
||||
disable_ads: bool = False
|
||||
geoip: bool = False
|
||||
adaptor_arguments: Optional[Dict] = None
|
||||
additional_arguments: Optional[Dict] = None
|
||||
selector_config: Optional[Dict] = None
|
||||
additional_args: Optional[Dict] = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Custom validation after msgspec validation"""
|
||||
@@ -136,10 +136,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
|
||||
self.cookies = []
|
||||
if self.solve_cloudflare and self.timeout < 60_000:
|
||||
self.timeout = 60_000
|
||||
if not self.adaptor_arguments:
|
||||
self.adaptor_arguments = {}
|
||||
if not self.additional_arguments:
|
||||
self.additional_arguments = {}
|
||||
if not self.selector_config:
|
||||
self.selector_config = {}
|
||||
if not self.additional_args:
|
||||
self.additional_args = {}
|
||||
|
||||
|
||||
def validate(params, model):
|
||||
|
||||
Reference in New Issue
Block a user