freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding the argument automatch_domain to all fetchers

Browse Source
This commit is contained in:
Karim shoair
2024-11-09 15:38:27 +02:00
parent 03091beb8b
commit b90c8adb02
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/custom.py
+11 -2
View File
@@ -27,10 +27,11 @@ class Response:
@property @property
def adaptor(self) -> Union[Adaptor, None]: def adaptor(self) -> Union[Adaptor, None]:
"""Generate Adaptor instance from this response if possible, otherwise return None""" """Generate Adaptor instance from this response if possible, otherwise return None"""
automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
if self.content: if self.content:
return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments) return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
elif self.text: elif self.text:
return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments) return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
return None return None
def __repr__(self): def __repr__(self):
@@ -41,6 +42,7 @@ class BaseFetcher:
def __init__( def __init__(
self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True, self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True, storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
automatch_domain: Optional[str] = None,
): ):
"""Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
are detected and passed automatically from the Fetcher based on the response for accessibility. are detected and passed automatically from the Fetcher based on the response for accessibility.
@@ -53,6 +55,8 @@ class BaseFetcher:
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info. :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
:param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class. :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
If empty, default values will be used. If empty, default values will be used.
:param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
Otherwise, the domain of the request is used by default.
:param debug: Enable debug mode :param debug: Enable debug mode
""" """
# Adaptor class parameters # Adaptor class parameters
@@ -67,6 +71,11 @@ class BaseFetcher:
) )
# If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
setup_basic_logging(level='debug' if debug else 'info') setup_basic_logging(level='debug' if debug else 'info')
if automatch_domain:
if type(automatch_domain) is not str:
logging.warning('[Ignored] The argument "automatch_domain" must be of string type')
else:
self.adaptor_arguments.update({'automatch_domain': automatch_domain})
def check_if_engine_usable(engine: Callable) -> Union[Callable, None]: def check_if_engine_usable(engine: Callable) -> Union[Callable, None]: