feat(unified import): Make one import for all fetchers

Why I didn't think of that from the start? IDK

scrapling/engines/toolbelt/custom.py

@@ -105,41 +105,77 @@ class Response(Adaptor):
 
 
 class BaseFetcher:
-    def __init__(
+    __slots__ = ()
-            self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
+    huge_tree: bool = True
-            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,
+    auto_match: Optional[bool] = True
-            automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
+    storage: Any = SQLiteStorageSystem
-    ):
+    keep_cdata: Optional[bool] = False
-        """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
+    storage_args: Optional[Dict] = None
-        are detected and passed automatically from the Fetcher based on the response for accessibility.
+    keep_comments: Optional[bool] = False
+    automatch_domain: Optional[str] = None
+    parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',)  # Left open for the user
 
-        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
+    def __init__(self, *args, **kwargs):
-            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
+        # For backward-compatibility before 0.2.99
-        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+        args_str = ", ".join(args) or ''
-        :param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
+        kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
-        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
+        if args_str:
-            priority over all auto-match related arguments/functions in the class.
+            args_str += ', '
-        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
+
-        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
+        log.warning(f'This logic is deprecated now and it will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
-            If empty, default values will be used.
+        pass
-        :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
+
-            Otherwise, the domain of the request is used by default.
+    @classmethod
+    def display_config(cls):
+        return dict(
+            huge_tree=cls.huge_tree,
+            keep_comments=cls.keep_comments,
+            keep_cdata=cls.keep_cdata,
+            auto_match=cls.auto_match,
+            storage=cls.storage,
+            storage_args=cls.storage_args,
+            automatch_domain=cls.automatch_domain,
+        )
+
+    @classmethod
+    def configure(cls, **kwargs):
+        """Setup multiple arguments for the parser at once
+
+        :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
         """
+        for key, value in kwargs.items():
+            key = key.strip().lower()
+            if hasattr(cls, key):
+                if key in cls.parser_keywords:
+                    setattr(cls, key, value)
+                else:
+                    # Yup, no fun allowed LOL
+                    raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+            else:
+                raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+
+        if not kwargs:
+            raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
+
+    @classmethod
+    def _generate_parser_arguments(cls) -> Dict:
         # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
-        self.adaptor_arguments = dict(
+        parser_arguments = dict(
-            huge_tree=huge_tree,
+            huge_tree=cls.huge_tree,
-            keep_comments=keep_comments,
+            keep_comments=cls.keep_comments,
-            keep_cdata=keep_cdata,
+            keep_cdata=cls.keep_cdata,
-            auto_match=auto_match,
+            auto_match=cls.auto_match,
-            storage=storage,
+            storage=cls.storage,
-            storage_args=storage_args
+            storage_args=cls.storage_args
         )
-        if automatch_domain:
+        if cls.automatch_domain:
-            if type(automatch_domain) is not str:
+            if type(cls.automatch_domain) is not str:
                 log.warning('[Ignored] The argument "automatch_domain" must be of string type')
             else:
-                self.adaptor_arguments.update({'automatch_domain': automatch_domain})
+                parser_arguments.update({'automatch_domain': cls.automatch_domain})
+
+        return parser_arguments
 
 
 class StatusText:

scrapling/fetchers.py

@@ -10,8 +10,9 @@ class Fetcher(BaseFetcher):
 
     Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
     """
+    @classmethod
     def get(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
@@ -25,12 +26,13 @@ class Fetcher(BaseFetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).get(**kwargs)
         return response_object
 
+    @classmethod
     def post(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
@@ -44,12 +46,13 @@ class Fetcher(BaseFetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).post(**kwargs)
         return response_object
 
+    @classmethod
     def put(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
@@ -64,12 +67,13 @@ class Fetcher(BaseFetcher):
 
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).put(**kwargs)
         return response_object
 
+    @classmethod
     def delete(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
@@ -83,14 +87,15 @@ class Fetcher(BaseFetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(**kwargs)
         return response_object
 
 
 class AsyncFetcher(Fetcher):
+    @classmethod
     async def get(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
@@ -104,12 +109,13 @@ class AsyncFetcher(Fetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_get(**kwargs)
         return response_object
 
+    @classmethod
     async def post(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
@@ -123,12 +129,13 @@ class AsyncFetcher(Fetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs)
         return response_object
 
+    @classmethod
     async def put(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
@@ -142,12 +149,13 @@ class AsyncFetcher(Fetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_put(**kwargs)
         return response_object
 
+    @classmethod
     async def delete(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
+            cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
@@ -161,7 +169,7 @@ class AsyncFetcher(Fetcher):
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        adaptor_arguments = tuple(cls._generate_parser_arguments().items())
         response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_delete(**kwargs)
         return response_object
 
@@ -172,8 +180,9 @@ class StealthyFetcher(BaseFetcher):
      It works as real browsers passing almost all online tests/protections based on Camoufox.
      Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
     """
+    @classmethod
     def fetch(
-            self, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
+            cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
             block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
             wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
@@ -226,12 +235,13 @@ class StealthyFetcher(BaseFetcher):
             extra_headers=extra_headers,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
-            adaptor_arguments=self.adaptor_arguments,
+            adaptor_arguments=cls._generate_parser_arguments(),
         )
         return engine.fetch(url)
 
+    @classmethod
     async def async_fetch(
-            self, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
+            cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
             block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
             wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
@@ -284,7 +294,7 @@ class StealthyFetcher(BaseFetcher):
             extra_headers=extra_headers,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
-            adaptor_arguments=self.adaptor_arguments,
+            adaptor_arguments=cls._generate_parser_arguments(),
         )
         return await engine.async_fetch(url)
 
@@ -305,8 +315,9 @@ class PlayWrightFetcher(BaseFetcher):
 
     > Note that these are the main options with PlayWright but it can be mixed together.
     """
+    @classmethod
     def fetch(
-            self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
+            cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,
             page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
             hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
@@ -361,12 +372,13 @@ class PlayWrightFetcher(BaseFetcher):
             nstbrowser_config=nstbrowser_config,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
-            adaptor_arguments=self.adaptor_arguments,
+            adaptor_arguments=cls._generate_parser_arguments(),
         )
         return engine.fetch(url)
 
+    @classmethod
     async def async_fetch(
-            self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
+            cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,
             page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
             hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
@@ -421,12 +433,13 @@ class PlayWrightFetcher(BaseFetcher):
             nstbrowser_config=nstbrowser_config,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
-            adaptor_arguments=self.adaptor_arguments,
+            adaptor_arguments=cls._generate_parser_arguments(),
         )
         return await engine.async_fetch(url)
 
 
 class CustomFetcher(BaseFetcher):
-    def fetch(self, url: str, browser_engine, **kwargs) -> Response:
+    @classmethod
-        engine = check_if_engine_usable(browser_engine)(adaptor_arguments=self.adaptor_arguments, **kwargs)
+    def fetch(cls, url: str, browser_engine, **kwargs) -> Response:
+        engine = check_if_engine_usable(browser_engine)(adaptor_arguments=cls._generate_parser_arguments(), **kwargs)
         return engine.fetch(url)