Big structure changes (check commit description)

- Moved most of the parser functions/files to the core package.
- Converted tools file to a package and made separate files for similar functions.
- Now all fetcher engines return a Response object
- Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing.
- I added a new custom fetcher so the user can create and use an engine.
- More...

scrapling/engines/camo.py

@@ -1,26 +1,28 @@
 import logging
-from scrapling._types import Union, Callable, Optional
+from scrapling.core._types import Union, Callable, Optional, Dict
 
-from .tools import check_type_validity, get_os_name, generate_convincing_referer
+from scrapling.engines.toolbelt import (
+    Response,
+    do_nothing,
+    get_os_name,
+    check_type_validity,
+    generate_convincing_referer,
+)
 
 from camoufox.sync_api import Camoufox
 
 
-def _do_nothing(page):
-    # Anything
-    return page
-
-
 class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, str] = True,
-            block_images: Optional[bool] = True,
+            block_images: Optional[bool] = False,
             block_webrtc: Optional[bool] = False,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
-            page_action: Callable = _do_nothing,
+            page_action: Callable = do_nothing,
             wait_selector: Optional[str] = None,
             wait_selector_state: str = 'attached',
+            adaptor_arguments: Dict = None
     ):
         self.headless = headless
         self.block_images = bool(block_images)
@@ -30,23 +32,24 @@ class CamoufoxEngine:
         if callable(page_action):
             self.page_action = page_action
         else:
-            self.page_action = _do_nothing
+            self.page_action = do_nothing
             logging.error('[Ignored] Argument "page_action" must be callable')
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
-    def fetch(self, url: str):
+    def fetch(self, url: str) -> Response:
         with Camoufox(
                 headless=self.headless,
-                block_images=self.block_images,
+                block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
                 os=get_os_name(),
                 block_webrtc=self.block_webrtc,
         ) as browser:
             page = browser.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
-            page.goto(url, referer=generate_convincing_referer(url))
+            res = page.goto(url, referer=generate_convincing_referer(url))
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
@@ -58,6 +61,24 @@ class CamoufoxEngine:
                 waiter = page.locator(self.wait_selector)
                 waiter.wait_for(state=self.wait_selector_state)
 
-            html = page.content()
+            content_type = res.headers.get('content-type', '')
+            # Parse charset from content-type
+            encoding = 'utf-8'  # default encoding
+            if 'charset=' in content_type.lower():
+                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+
+            response = Response(
+                url=res.url,
+                text=res.text(),
+                content=res.body(),
+                status=res.status,
+                reason=res.status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
+                headers=res.all_headers(),
+                request_headers=res.request.all_headers(),
+                adaptor_arguments=self.adaptor_arguments
+            )
             page.close()
-        return html
+
+        return response