freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: huge change, many features/class got a better naming

Browse Source 
- `Adaptor` became `Selector`
- `Adaptors` became `Selectors`
- `auto_match` argument/feature became `adaptive`
- `adaptor_arguments` argument became `selector_config`
- `automatch_domain` argument became `adaptive_domain`
- `additional_arguments` argument became `additional_args`
- `storage_adaptors` file became just `storage`
This commit is contained in:
Karim shoair
2025-07-29 04:20:23 +03:00
parent 0c649987f8
commit 264ae02aa7
22 changed files with 250 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+20 -20
View File
@@ -70,8 +70,8 @@ class StealthySession:
"os_randomize",
"disable_ads",
"geoip",
"adaptor_arguments",
"additional_arguments",
"selector_config",
"additional_args",
"playwright",
"browser",
"context",
@@ -105,8 +105,8 @@ class StealthySession:
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
@@ -136,8 +136,8 @@ class StealthySession:
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
params = {
@@ -163,8 +163,8 @@ class StealthySession:
"os_randomize": os_randomize,
"disable_ads": disable_ads,
"geoip": geoip,
"adaptor_arguments": adaptor_arguments,
"additional_arguments": additional_arguments,
"selector_config": selector_config,
"additional_args": additional_args,
}
config = validate(params, CamoufoxConfig)
@@ -190,14 +190,14 @@ class StealthySession:
self.os_randomize = config.os_randomize
self.disable_ads = config.disable_ads
self.geoip = config.geoip
self.adaptor_arguments = config.adaptor_arguments
self.additional_arguments = config.additional_arguments
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.playwright: Optional[Playwright] = None
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
self.adaptor_arguments = config.adaptor_arguments
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
@@ -223,7 +223,7 @@ class StealthySession:
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self.os_randomize else get_os_name(),
"user_data_dir": "",
**self.additional_arguments,
**self.additional_args,
}
)
@@ -433,7 +433,7 @@ class StealthySession:
page_info.page.wait_for_timeout(self.wait)
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
@@ -482,8 +482,8 @@ class AsyncStealthySession(StealthySession):
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
@@ -513,8 +513,8 @@ class AsyncStealthySession(StealthySession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
super().__init__(
max_pages,
@@ -539,8 +539,8 @@ class AsyncStealthySession(StealthySession):
os_randomize,
disable_ads,
geoip,
adaptor_arguments,
additional_arguments,
selector_config,
additional_args,
)
self.playwright: Optional[AsyncPlaywright] = None
self.context: Optional[AsyncBrowserContext] = None
@@ -731,7 +731,7 @@ class AsyncStealthySession(StealthySession):
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
scrapling/engines/_browsers/_controllers.py
+10 -10
View File
@@ -70,7 +70,7 @@ class DynamicSession:
"context",
"page_pool",
"_closed",
"adaptor_arguments",
"selector_config",
"page_action",
"launch_options",
"context_options",
@@ -100,7 +100,7 @@ class DynamicSession:
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
@@ -125,7 +125,7 @@ class DynamicSession:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
params = {
@@ -143,7 +143,7 @@ class DynamicSession:
"extra_headers": extra_headers,
"useragent": useragent,
"timeout": timeout,
"adaptor_arguments": adaptor_arguments,
"selector_config": selector_config,
"disable_resources": disable_resources,
"wait_selector": wait_selector,
"cookies": cookies,
@@ -177,7 +177,7 @@ class DynamicSession:
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
self.adaptor_arguments = config.adaptor_arguments
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
@@ -370,7 +370,7 @@ class DynamicSession:
# Create response object
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
@@ -417,7 +417,7 @@ class AsyncDynamicSession(DynamicSession):
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
@@ -443,7 +443,7 @@ class AsyncDynamicSession(DynamicSession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
super().__init__(
@@ -467,7 +467,7 @@ class AsyncDynamicSession(DynamicSession):
cookies,
network_idle,
wait_selector_state,
adaptor_arguments,
selector_config,
)
self.playwright: Optional[AsyncPlaywright] = None
@@ -623,7 +623,7 @@ class AsyncDynamicSession(DynamicSession):
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
scrapling/engines/_browsers/_validators.py
+9 -9
View File
@@ -39,7 +39,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
cookies: Optional[List[Dict]] = None
network_idle: bool = False
wait_selector_state: SelectorWaitStates = "attached"
adaptor_arguments: Optional[Dict] = None
selector_config: Optional[Dict] = None
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -57,8 +57,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
self.__validate_cdp(self.cdp_url)
if not self.cookies:
self.cookies = []
if not self.adaptor_arguments:
self.adaptor_arguments = {}
if not self.selector_config:
self.selector_config = {}
@staticmethod
def __validate_cdp(cdp_url):
@@ -105,8 +105,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
adaptor_arguments: Optional[Dict] = None
additional_arguments: Optional[Dict] = None
selector_config: Optional[Dict] = None
additional_args: Optional[Dict] = None
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -136,10 +136,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.cookies = []
if self.solve_cloudflare and self.timeout < 60_000:
self.timeout = 60_000
if not self.adaptor_arguments:
self.adaptor_arguments = {}
if not self.additional_arguments:
self.additional_arguments = {}
if not self.selector_config:
self.selector_config = {}
if not self.additional_args:
self.additional_args = {}
def validate(params, model):
scrapling/engines/static.py
+12 -18
View File
@@ -63,7 +63,7 @@ class FetcherSession:
max_redirects: int = 30,
verify: bool = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
adaptor_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
):
"""
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
@@ -81,7 +81,7 @@ class FetcherSession:
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class.
:param selector_config: Arguments passed when creating the final Selector class.
"""
self.default_impersonate = impersonate
self.stealth = stealthy_headers
@@ -97,7 +97,7 @@ class FetcherSession:
self.default_verify = verify
self.default_cert = cert
self.default_http3 = http3
self.adaptor_arguments = adaptor_arguments or {}
self.selector_config = selector_config or {}
self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] = None
@@ -260,7 +260,7 @@ class FetcherSession:
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
adaptor_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
) -> Response:
"""
Perform an HTTP request using the configured session.
@@ -270,7 +270,7 @@ class FetcherSession:
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class.
:param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._curl_session
@@ -286,9 +286,7 @@ class FetcherSession:
try:
response = session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(
response, adaptor_arguments
)
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e:
if attempt < max_retries - 1:
log.error(
@@ -307,7 +305,7 @@ class FetcherSession:
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
adaptor_arguments: Optional[Dict] = None,
selector_config: Optional[Dict] = None,
) -> Response:
"""
Perform an HTTP request using the configured session.
@@ -317,7 +315,7 @@ class FetcherSession:
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class.
:param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._async_curl_session
@@ -335,9 +333,7 @@ class FetcherSession:
try:
response = await session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(
response, adaptor_arguments
)
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e:
if attempt < max_retries - 1:
log.error(
@@ -373,9 +369,7 @@ class FetcherSession:
"""
stealth = self.stealth if stealth is None else stealth
adaptor_arguments = (
kwargs.pop("adaptor_arguments", {}) or self.adaptor_arguments
)
selector_config = kwargs.pop("selector_config", {}) or self.selector_config
max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries)
retry_delay = self.get_with_precedence(
kwargs, "retry_delay", self.default_retry_delay
@@ -383,12 +377,12 @@ class FetcherSession:
request_args = self._merge_request_args(stealth=stealth, **kwargs)
if self._curl_session:
return self.__make_request(
method, request_args, max_retries, retry_delay, adaptor_arguments
method, request_args, max_retries, retry_delay, selector_config
)
elif self._async_curl_session:
# The returned value is a Coroutine
return self.__make_async_request(
method, request_args, max_retries, retry_delay, adaptor_arguments
method, request_args, max_retries, retry_delay, selector_config
)
raise RuntimeError("No active session available.")
scrapling/engines/toolbelt/convertor.py
+1 -1
View File
@@ -239,7 +239,7 @@ class ResponseFactory:
:param response: `curl_cffi` response object
:param parser_arguments: Additional arguments to be passed to the `Response` object constructor.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return Response(
url=response.url,
scrapling/engines/toolbelt/custom.py
+20 -22
View File
@@ -15,7 +15,7 @@ from scrapling.core._types import (
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.parser import Selector, SQLiteStorageSystem
class ResponseEncoding:
@@ -97,7 +97,7 @@ class ResponseEncoding:
return cls.__DEFAULT_ENCODING
class Response(Adaptor):
class Response(Selector):
"""This class is returned by all engines as a way to unify response type between different libraries."""
def __init__(
@@ -113,9 +113,9 @@ class Response(Adaptor):
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**adaptor_arguments: Dict,
**selector_config: Dict,
):
automatch_domain = adaptor_arguments.pop("automatch_domain", None)
adaptive_domain = selector_config.pop("adaptive_domain", None)
self.status = status
self.reason = reason
self.cookies = cookies
@@ -126,12 +126,10 @@ class Response(Adaptor):
super().__init__(
text=text,
body=body,
url=automatch_domain or url,
url=adaptive_domain or url,
encoding=encoding,
**adaptor_arguments,
**selector_config,
)
# For backward compatibility
self.adaptor = self
# For easier debugging while working from a Python shell
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
@@ -144,20 +142,20 @@ class Response(Adaptor):
class BaseFetcher:
__slots__ = ()
huge_tree: bool = True
auto_match: Optional[bool] = False
adaptive: Optional[bool] = False
storage: Any = SQLiteStorageSystem
keep_cdata: Optional[bool] = False
storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None
adaptive_domain: Optional[str] = None
parser_keywords: Tuple = (
"huge_tree",
"auto_match",
"adaptive",
"storage",
"keep_cdata",
"storage_args",
"keep_comments",
"automatch_domain",
"adaptive_domain",
) # Left open for the user
def __init__(self, *args, **kwargs):
@@ -178,17 +176,17 @@ class BaseFetcher:
huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
adaptive=cls.adaptive,
storage=cls.storage,
storage_args=cls.storage_args,
automatch_domain=cls.automatch_domain,
adaptive_domain=cls.adaptive_domain,
)
@classmethod
def configure(cls, **kwargs):
"""Set multiple arguments for the parser at once globally
:param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
:param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, adaptive, storage, storage_args, adaptive_domain
"""
for key, value in kwargs.items():
key = key.strip().lower()
@@ -212,23 +210,23 @@ class BaseFetcher:
@classmethod
def _generate_parser_arguments(cls) -> Dict:
# Adaptor class parameters
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
# Selector class parameters
# I won't validate Selector's class parameters here again, I will leave it to be validated later
parser_arguments = dict(
huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
adaptive=cls.adaptive,
storage=cls.storage,
storage_args=cls.storage_args,
)
if cls.automatch_domain:
if type(cls.automatch_domain) is not str:
if cls.adaptive_domain:
if type(cls.adaptive_domain) is not str:
log.warning(
'[Ignored] The argument "automatch_domain" must be of string type'
'[Ignored] The argument "adaptive_domain" must be of string type'
)
else:
parser_arguments.update({"automatch_domain": cls.automatch_domain})
parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
return parser_arguments