From 231aeb06164fa0c6d3377831e9caf1dfd7e11b9e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 26 Oct 2024 21:30:44 +0300
Subject: [PATCH 001/118] Dropping Python 3.7 from the tests for next version

---
 .github/workflows/tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 0acf1b8..77dd783 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -12,10 +12,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.7"
-          os: ubuntu-latest
-          env:
-            TOXENV: py
         - python-version: "3.8"
           os: ubuntu-latest
           env:
@@ -36,6 +32,10 @@ jobs:
           os: ubuntu-latest
           env:
             TOXENV: py
+        - python-version: "3.13"
+          os: ubuntu-latest
+          env:
+            TOXENV: py
 
     steps:
     - uses: actions/checkout@v4

From 7515f803bc2fc016cc8dda9462d7205f05a32bae Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 26 Oct 2024 21:31:31 +0300
Subject: [PATCH 002/118] Be more specific about tests

---
 tests/{test_all_functions.py => test_parser_functions.py} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename tests/{test_all_functions.py => test_parser_functions.py} (99%)

diff --git a/tests/test_all_functions.py b/tests/test_parser_functions.py
similarity index 99%
rename from tests/test_all_functions.py
rename to tests/test_parser_functions.py
index 19202bb..80c8f0f 100644
--- a/tests/test_all_functions.py
+++ b/tests/test_parser_functions.py
@@ -331,6 +331,6 @@ class TestParser(unittest.TestCase):
         self.assertLess(end_time - start_time, 0.1)
 
 
-# Use `coverage run -m unittest --verbose tests/test_all_functions.py` instead for the coverage report
+# Use `coverage run -m unittest --verbose tests/test_parser_functions.py` instead for the coverage report
 # if __name__ == '__main__':
 #     unittest.main(verbosity=2)

From 878fe0d37375bad6164d72c167abba42d4e3f14a Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 26 Oct 2024 21:31:57 +0300
Subject: [PATCH 003/118] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 99132aa..7c50908 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -15,7 +15,7 @@ configfile: pytest.ini
 plugins: cov-5.0.0, anyio-4.6.0
 collected 16 items
 
-tests/test_all_functions.py ................          [100%]
+tests/test_parser_functions.py ................          [100%]
 
 =============================== 16 passed in 0.22s ================================
 ```

From 327281c6643103af43eeebc7e9487fce610ac9b0 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 26 Oct 2024 21:32:57 +0300
Subject: [PATCH 004/118] Making structure clearer to help with the next
 version

---
 scrapling/{utils.py => utils/__init__.py} | 60 -----------------------
 1 file changed, 60 deletions(-)
 rename scrapling/{utils.py => utils/__init__.py} (58%)

diff --git a/scrapling/utils.py b/scrapling/utils/__init__.py
similarity index 58%
rename from scrapling/utils.py
rename to scrapling/utils/__init__.py
index 7eea9fa..3defaf4 100644
--- a/scrapling/utils.py
+++ b/scrapling/utils/__init__.py
@@ -1,8 +1,6 @@
 import re
-import os
 import logging
 from itertools import chain
-from logging import handlers
 # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
 from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
 
@@ -45,64 +43,6 @@ def _is_iterable(s: Any):
     return isinstance(s, (list, tuple,))
 
 
-@cache(None, typed=True)
-class _Logger(object):
-    # I will leave this class here for now in case I decide I want to come back to use it :)
-    __slots__ = ('console_logger', 'logger_file_path',)
-    levels = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warning': logging.WARNING,
-        'error': logging.ERROR,
-        'critical': logging.CRITICAL
-    }
-
-    def __init__(self, filename: str = 'debug.log', level: str = 'debug', when: str = 'midnight', backcount: int = 1):
-        os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True)
-        format_str = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
-
-        # on-screen output
-        lvl = self.levels[level.lower()]
-        self.console_logger = logging.getLogger('Scrapling')
-        self.console_logger.setLevel(lvl)
-        console_handler = logging.StreamHandler()
-        console_handler.setLevel(lvl)
-        console_handler.setFormatter(format_str)
-        self.console_logger.addHandler(console_handler)
-
-        if lvl == logging.DEBUG:
-            filename = os.path.join(os.path.dirname(__file__), 'logs', filename)
-            self.logger_file_path = filename
-            # Automatically generates the logging file at specified intervals
-            file_handler = handlers.TimedRotatingFileHandler(
-                # If more than (backcount+1) existed, oldest logs will be deleted
-                filename=filename, when=when, backupCount=backcount, encoding='utf-8'
-            )
-            file_handler.setLevel(lvl)
-            file_handler.setFormatter(format_str)
-            # This for the logger when it appends the date to the new log
-            file_handler.namer = lambda name: name.replace(".log", "") + ".log"
-            self.console_logger.addHandler(file_handler)
-            self.debug(f'Debug log path: {self.logger_file_path}')
-        else:
-            self.logger_file_path = None
-
-    def debug(self, message: str) -> None:
-        self.console_logger.debug(message)
-
-    def info(self, message: str) -> None:
-        self.console_logger.info(message)
-
-    def warning(self, message: str) -> None:
-        self.console_logger.warning(message)
-
-    def error(self, message: str) -> None:
-        self.console_logger.error(message)
-
-    def critical(self, message: str) -> None:
-        self.console_logger.critical(message)
-
-
 class _StorageTools:
     @staticmethod
     def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:

From 427f43eb969830e233725efcc2169aecf8e0cf38 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 01:15:41 +0300
Subject: [PATCH 005/118] Update MANIFEST.in

---
 MANIFEST.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index c69a5e6..736106d 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,8 @@
 include LICENSE
 include *.db
+include *.js
 include scrapling/*.db
+include scrapling/*.db*
 include scrapling/py.typed
 
 recursive-exclude * __pycache__

From e485d51b43c4061bf9d2a47ed5e8b2abff9faa77 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 01:16:38 +0300
Subject: [PATCH 006/118] Update ROADMAP.md

---
 ROADMAP.md | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/ROADMAP.md b/ROADMAP.md
index f52f858..5847c87 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -1,13 +1,14 @@
 ## TODOs
-- Add more tests and increase the code coverage.
-- Structure the tests folder in a better way.
-- Add more documentation.
-- Add the browsing ability.
-- Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.
-- Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
-- Need to add more functionality to `AttributesHandler` and more navigation functions to `Adaptor` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
-- Add `.filter` method to `Adaptors` object and other similar methods.
-- Add functionality to automatically detect pagination URLs
-- Add the ability to auto-detect schemas in pages and manipulate them
-- Add ability to generate a regex from a group of elements (Like for all href attributes)
+- [ ] Add more tests and increase the code coverage.
+- [ ] Structure the tests folder in a better way.
+- [ ] Add more documentation.
+- [x] Add the browsing ability.
+- [ ] Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.
+- [ ] Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
+- [ ] Need to add more functionality to `AttributesHandler` and more navigation functions to `Adaptor` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
+- [ ] Add `.filter` method to `Adaptors` object and other similar methods.
+- [ ] Add functionality to automatically detect pagination URLs
+- [ ] Add the ability to auto-detect schemas in pages and manipulate them.
+- [ ] Add `analyzer` ability that tries to learn about the page through meta elements and return what it learned
+- [ ] Add ability to generate a regex from a group of elements (Like for all href attributes)
 - 
\ No newline at end of file

From 1ca2b6cfe43516be2fe6e36602125a5e52a37f34 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 01:31:10 +0300
Subject: [PATCH 007/118] Combing type definitions in one file

---
 scrapling/_types.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 scrapling/_types.py

diff --git a/scrapling/_types.py b/scrapling/_types.py
new file mode 100644
index 0000000..ca6769c
--- /dev/null
+++ b/scrapling/_types.py
@@ -0,0 +1,25 @@
+"""
+Type definitions for type checking purposes.
+"""
+
+from typing import (
+    Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING
+)
+
+try:
+    from scrapling._types import Protocol
+except ImportError:
+    # Added in Python 3.8
+    Protocol = object
+
+try:
+    from scrapling._types import SupportsIndex
+except ImportError:
+    # 'SupportsIndex' got added in Python 3.8
+    SupportsIndex = None
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+else:
+    Self = object

From 23c4181ff239fb306d7031d7266254b170148ea1 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 01:32:18 +0300
Subject: [PATCH 008/118] Using the new types file

---
 scrapling/custom_types.py     |  2 +-
 scrapling/parser.py           |  7 +------
 scrapling/storage_adaptors.py |  2 +-
 scrapling/translator.py       | 12 +-----------
 4 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/scrapling/custom_types.py b/scrapling/custom_types.py
index 0c5fb67..5ff3b5f 100644
--- a/scrapling/custom_types.py
+++ b/scrapling/custom_types.py
@@ -1,9 +1,9 @@
 import re
 from types import MappingProxyType
 from collections.abc import Mapping
-from typing import Dict, List, Union, Pattern
 
 from scrapling.utils import _is_iterable, flatten
+from scrapling._types import Dict, List, Union, Pattern
 
 from orjson import loads, dumps
 from w3lib.html import replace_entities as _replace_entities
diff --git a/scrapling/parser.py b/scrapling/parser.py
index a517112..771d08a 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,17 +1,12 @@
 import os
 from difflib import SequenceMatcher
-from typing import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator
-try:
-    from typing import SupportsIndex
-except ImportError:
-    # 'SupportsIndex' got added in Python 3.8
-    SupportsIndex = None
 
 from scrapling.translator import HTMLTranslator
 from scrapling.mixins import SelectorsGeneration
 from scrapling.custom_types import TextHandler, AttributesHandler
 from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
 from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
+from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
 
 from lxml import etree, html
 from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
diff --git a/scrapling/storage_adaptors.py b/scrapling/storage_adaptors.py
index ec88925..d9018a3 100644
--- a/scrapling/storage_adaptors.py
+++ b/scrapling/storage_adaptors.py
@@ -4,8 +4,8 @@ import logging
 import threading
 from hashlib import sha256
 from abc import ABC, abstractmethod
-from typing import Dict, Optional, Union
 
+from scrapling._types import Dict, Optional, Union
 from scrapling.utils import _StorageTools, cache
 
 from lxml import html
diff --git a/scrapling/translator.py b/scrapling/translator.py
index ec6db86..77679a6 100644
--- a/scrapling/translator.py
+++ b/scrapling/translator.py
@@ -9,24 +9,14 @@ which will be important in future releases but most importantly...
 import re
 
 from w3lib.html import HTML5_WHITESPACE
-from typing import TYPE_CHECKING, Any, Optional
-try:
-    from typing import Protocol
-except ImportError:
-    # Added in Python 3.8
-    Protocol = object
-
 from scrapling.utils import cache
+from scrapling._types import Any, Optional, Protocol, Self
 
 from cssselect.xpath import ExpressionError
 from cssselect.xpath import XPathExpr as OriginalXPathExpr
 from cssselect import HTMLTranslator as OriginalHTMLTranslator
 from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
 
-if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
-
 
 regex = f"[{HTML5_WHITESPACE}]+"
 replace_html5_whitespaces = re.compile(regex).sub

From 6eeca3daa706c5878bf10087a8c15c62c8f3159f Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 01:32:53 +0300
Subject: [PATCH 009/118] Going back to single utils file and using the new
 types file

---
 scrapling/{utils/__init__.py => utils.py} | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
 rename scrapling/{utils/__init__.py => utils.py} (98%)

diff --git a/scrapling/utils/__init__.py b/scrapling/utils.py
similarity index 98%
rename from scrapling/utils/__init__.py
rename to scrapling/utils.py
index 3defaf4..5f8efc9 100644
--- a/scrapling/utils/__init__.py
+++ b/scrapling/utils.py
@@ -4,9 +4,10 @@ from itertools import chain
 # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
 from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
 
-from typing import Dict, Iterable, Any
+from scrapling._types import Dict, Iterable, Any
 
 from lxml import html
+
 html_forbidden = {html.HtmlComment, }
 logging.basicConfig(
         level=logging.ERROR,

From 8f8d08b5682131e3de6378046e1921d4fc3638fc Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:11:33 +0300
Subject: [PATCH 010/118] Adding browsing engines

---
 scrapling/engines/__init__.py |   6 +
 scrapling/engines/camo.py     |  63 ++++++++++
 scrapling/engines/pw.py       | 209 ++++++++++++++++++++++++++++++++++
 scrapling/engines/static.py   |  53 +++++++++
 scrapling/engines/tools.py    | 174 ++++++++++++++++++++++++++++
 5 files changed, 505 insertions(+)
 create mode 100644 scrapling/engines/__init__.py
 create mode 100644 scrapling/engines/camo.py
 create mode 100644 scrapling/engines/pw.py
 create mode 100644 scrapling/engines/static.py
 create mode 100644 scrapling/engines/tools.py

diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py
new file mode 100644
index 0000000..89b5d62
--- /dev/null
+++ b/scrapling/engines/__init__.py
@@ -0,0 +1,6 @@
+from .camo import CamoufoxEngine
+from .static import StaticEngine
+from .pw import PlaywrightEngine, DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
+from .tools import check_if_engine_usable
+
+__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
new file mode 100644
index 0000000..26464ab
--- /dev/null
+++ b/scrapling/engines/camo.py
@@ -0,0 +1,63 @@
+import logging
+from scrapling._types import Union, Callable, Optional
+
+from .tools import check_type_validity, get_os_name, generate_convincing_referer
+
+from camoufox.sync_api import Camoufox
+
+
+def _do_nothing(page):
+    # Anything
+    return page
+
+
+class CamoufoxEngine:
+    def __init__(
+            self, headless: Union[bool, str] = True,
+            block_images: Optional[bool] = True,
+            block_webrtc: Optional[bool] = False,
+            network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000,
+            page_action: Callable = _do_nothing,
+            wait_selector: Optional[str] = None,
+            wait_selector_state: str = 'attached',
+    ):
+        self.headless = headless
+        self.block_images = bool(block_images)
+        self.block_webrtc = bool(block_webrtc)
+        self.network_idle = bool(network_idle)
+        self.timeout = check_type_validity(timeout, [int, float], 30000)
+        if callable(page_action):
+            self.page_action = page_action
+        else:
+            self.page_action = _do_nothing
+            logging.error('[Ignored] Argument "page_action" must be callable')
+
+        self.wait_selector = wait_selector
+        self.wait_selector_state = wait_selector_state
+
+    def fetch(self, url: str):
+        with Camoufox(
+                headless=self.headless,
+                block_images=self.block_images,
+                os=get_os_name(),
+                block_webrtc=self.block_webrtc,
+        ) as browser:
+            page = browser.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+            page.goto(url, referer=generate_convincing_referer(url))
+            page.wait_for_load_state(state="load")
+            page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                page.wait_for_load_state('networkidle')
+
+            page = self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                waiter.wait_for(state=self.wait_selector_state)
+
+            html = page.content()
+            page.close()
+        return html
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
new file mode 100644
index 0000000..db353fa
--- /dev/null
+++ b/scrapling/engines/pw.py
@@ -0,0 +1,209 @@
+import json
+import logging
+from scrapling._types import Union, Callable, Optional, List, Dict
+
+from .tools import check_type_validity, generate_headers, js_bypass_path, construct_websocket_url, generate_convincing_referer
+
+# Disable loading these resources for speed
+DEFAULT_DISABLED_RESOURCES = ['beacon', 'csp_report', 'font', 'image', 'imageset', 'media', 'object', 'texttrack', 'stylesheet', 'websocket']
+DEFAULT_STEALTH_FLAGS = [
+    # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
+    # Generally this will make the browser faster and less detectable
+    '--incognito', '--accept-lang=en-US', '--lang=en-US', '--no-pings', '--mute-audio', '--no-first-run', '--no-default-browser-check', '--disable-cloud-import',
+    '--disable-gesture-typing', '--disable-offer-store-unmasked-wallet-cards', '--disable-offer-upload-credit-cards', '--disable-print-preview', '--disable-voice-input',
+    '--disable-wake-on-wifi', '--disable-cookie-encryption', '--ignore-gpu-blocklist', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open',
+    '--prerender-from-omnibox=disabled', '--enable-web-bluetooth', '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
+    '--aggressive-cache-discard', '--disable-ipc-flooding-protection', '--disable-blink-features=AutomationControlled', '--test-type',
+    '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
+    '--disable-breakpad', '--disable-component-update', '--disable-domain-reliability', '--disable-sync', '--disable-client-side-phishing-detection',
+    '--disable-hang-monitor', '--disable-popup-blocking', '--disable-prompt-on-repost', '--metrics-recording-only', '--safebrowsing-disable-auto-update', '--password-store=basic',
+    '--autoplay-policy=no-user-gesture-required', '--use-mock-keychain', '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--webrtc-ip-handling-policy=disable_non_proxied_udp', '--disable-session-crashed-bubble', '--disable-crash-reporter', '--disable-dev-shm-usage', '--force-color-profile=srgb',
+    '--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-infobars',
+    '--hide-scrollbars', '--disable-renderer-backgrounding', '--font-render-hinting=none', '--disable-logging', '--enable-surface-synchronization',
+    '--run-all-compositor-stages-before-draw', '--disable-threaded-animation', '--disable-threaded-scrolling', '--disable-checker-imaging',
+    '--disable-new-content-rendering-timeout', '--disable-image-animation-resync', '--disable-partial-raster',
+    '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
+    '--disable-layer-tree-host-memory-pressure',
+    '--window-position=0,0',
+    '--disable-features=site-per-process',
+    '--disable-default-apps',
+    '--disable-component-extensions-with-background-pages',
+    '--disable-extensions',
+    # "--disable-reading-from-canvas",  # For Firefox
+    '--start-maximized'  # For headless check bypass
+]
+
+
+def _do_nothing(page):
+    # Anything
+    return page
+
+
+class PlaywrightEngine:
+    def __init__(
+            self, headless: Union[bool, str] = True,
+            disable_resources: Optional[List] = None,
+            useragent: Optional[str] = None,
+            network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000,
+            page_action: Callable = _do_nothing,
+            wait_selector: Optional[str] = None,
+            wait_selector_state: Optional[str] = 'attached',
+            stealth: bool = False,
+            hide_canvas: bool = True,
+            disable_webgl: bool = False,
+            cdp_url: Optional[str] = None,
+            nstbrowser_mode: bool = False,
+            nstbrowser_config: Optional[Dict] = None,
+    ):
+        self.headless = headless
+        self.disable_resources = disable_resources
+        self.network_idle = bool(network_idle)
+        self.stealth = bool(stealth)
+        self.hide_canvas = bool(hide_canvas)
+        self.disable_webgl = bool(disable_webgl)
+        self.cdp_url = cdp_url
+        self.useragent = useragent
+        self.timeout = check_type_validity(timeout, [int, float], 30000)
+        if callable(page_action):
+            self.page_action = page_action
+        else:
+            self.page_action = _do_nothing
+            logging.error('[Ignored] Argument "page_action" must be callable')
+
+        self.wait_selector = wait_selector
+        self.wait_selector_state = wait_selector_state
+        self.nstbrowser_mode = bool(nstbrowser_mode)
+        self.nstbrowser_config = nstbrowser_config
+
+    def _cdp_url_logic(self, flags: Optional[dict] = None):
+        cdp_url = self.cdp_url
+        if self.nstbrowser_mode:
+            if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
+                config = self.nstbrowser_config
+            else:
+                # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
+                query = {
+                    "once": True,
+                    "headless": True,
+                    "autoClose": True,
+                    "fingerprint": {
+                        "flags": {
+                            "timezone": "BasedOnIp",
+                            "screen": "Custom"
+                        },
+                        "platform": 'linux',  # support: windows, mac, linux
+                        "kernel": 'chromium',  # only support: chromium
+                        "kernelMilestone": '128',
+                        "hardwareConcurrency": 8,
+                        "deviceMemory": 8,
+                    },
+                }
+                if flags:
+                    query.update({
+                        "args": dict(zip(flags, [''] * len(flags))),  # browser args should be a dictionary
+                    })
+
+                config = {
+                    'config': json.dumps(query),
+                    # 'token': ''
+                }
+            cdp_url = construct_websocket_url(cdp_url, config)
+
+        return cdp_url
+
+    def fetch(self, url):
+        if not self.stealth:
+            from playwright.sync_api import sync_playwright
+        else:
+            from rebrowser_playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            # Handle the UserAgent early
+            if self.useragent:
+                extra_headers = {}
+                useragent = self.useragent
+            else:
+                extra_headers = generate_headers(browser_mode=True)
+                useragent = extra_headers.get('User-Agent')
+
+            # Prepare the flags before diving
+            flags = DEFAULT_STEALTH_FLAGS
+            if self.hide_canvas:
+                flags += ['--fingerprinting-canvas-image-data-noise']
+            if self.disable_webgl:
+                flags += ['--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2']
+
+            # Creating the browser
+            if self.cdp_url:
+                cdp_url = self._cdp_url_logic(flags if self.stealth else None)
+                browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
+            else:
+                if self.stealth:
+                    browser = p.chromium.launch(headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True)
+                else:
+                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'])
+
+            # Creating the context
+            if self.stealth:
+                context = browser.new_context(
+                    locale='en-US',
+                    is_mobile=False,
+                    has_touch=False,
+                    color_scheme='dark',  # Bypasses the 'prefersLightColor' check in creepjs
+                    user_agent=useragent,
+                    device_scale_factor=2,
+                    # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
+                    service_workers="allow",
+                    ignore_https_errors=True,
+                    extra_http_headers=extra_headers,
+                    screen={"width": 1920, "height": 1080},
+                    viewport={"width": 1920, "height": 1080},
+                    permissions=["geolocation", 'notifications'],
+                )
+            else:
+                context = browser.new_context(
+                    color_scheme='dark',
+                    user_agent=useragent,
+                    device_scale_factor=2,
+                    extra_http_headers=extra_headers
+                )
+
+            # Finally we are in business
+            page = context.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+            if self.stealth:
+                # Basic bypasses nothing fancy as I'm still working on it
+                # But with adding these bypasses to the above config, it bypasses many online tests like
+                # https://bot.sannysoft.com/
+                # https://kaliiiiiiiiii.github.io/brotector/
+                # https://pixelscan.net/
+                # https://iphey.com/
+                # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
+                # https://arh.antoinevastel.com/bots/areyouheadless/
+                # https://prescience-data.github.io/execution-monitor.html
+                page.add_init_script(path=js_bypass_path('webdriver_fully.js'))
+                page.add_init_script(path=js_bypass_path('window_chrome.js'))
+                page.add_init_script(path=js_bypass_path('navigator_plugins.js'))
+                page.add_init_script(path=js_bypass_path('pdf_viewer.js'))
+                page.add_init_script(path=js_bypass_path('notification_permission.js'))
+                page.add_init_script(path=js_bypass_path('screen_props.js'))
+                page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
+
+            page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
+            page.wait_for_load_state(state="load")
+            page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                page.wait_for_load_state('networkidle')
+
+            page = self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                waiter.wait_for(state=self.wait_selector_state)
+
+            html = page.content()
+            page.close()
+        return html
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
new file mode 100644
index 0000000..d4110e7
--- /dev/null
+++ b/scrapling/engines/static.py
@@ -0,0 +1,53 @@
+import logging
+from scrapling._types import Union, Optional, Dict
+
+from .tools import generate_convincing_referer, generate_headers
+
+import httpx
+
+
+class StaticEngine:
+    def __init__(
+            self,
+            follow_redirects: bool = True,
+            timeout: Optional[Union[int, float]] = None,
+    ):
+        self.timeout = timeout
+        self.follow_redirects = bool(follow_redirects)
+        self._extra_headers = generate_headers(browser_mode=False)
+
+    @staticmethod
+    def _headers_job(headers, url, stealth):
+        headers = headers or {}
+
+        # Validate headers
+        if not headers.get('user-agent') and not headers.get('User-Agent'):
+            headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
+            logging.info(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
+
+        if stealth:
+            extra_headers = generate_headers(browser_mode=False)
+            headers.update(extra_headers)
+            headers.update({'referer': generate_convincing_referer(url)})
+
+        return headers
+
+    def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return request.text
+
+    def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return request.text
+
+    def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return request.text
+
+    def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return request.text
diff --git a/scrapling/engines/tools.py b/scrapling/engines/tools.py
new file mode 100644
index 0000000..3d4564e
--- /dev/null
+++ b/scrapling/engines/tools.py
@@ -0,0 +1,174 @@
+import os
+import logging
+import inspect
+import platform
+from scrapling._types import Any, List, Type, Union, Optional
+from urllib.parse import urlparse, urlencode
+
+from tldextract import extract
+from browserforge.fingerprints import FingerprintGenerator
+from browserforge.headers import HeaderGenerator, Browser
+
+
+def generate_convincing_referer(url):
+    """
+    Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
+
+    >>> generate_convincing_referer('https://www.somewebsite.com/blah')
+    'https://www.google.com/search?q=somewebsite'
+
+    :param url: The URL you are about to fetch.
+    :return:
+    """
+    website_name = extract(url).domain
+    return f'https://www.google.com/search?q={website_name}'
+
+
+def check_if_engine_usable(engine):
+    if isinstance(engine, type):
+        raise TypeError("Expected an engine instance, not a class definition of the engine")
+
+    if hasattr(engine, 'fetch'):
+        fetch_function = getattr(engine, "fetch")
+        if callable(fetch_function):
+            if len(inspect.signature(fetch_function).parameters) > 0:
+                return engine
+            else:
+                raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
+        else:
+            raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
+    else:
+        raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
+
+
+def construct_websocket_url(base_url, query_params):
+    # Validate the base URL structure
+    try:
+        parsed = urlparse(base_url)
+
+        # Check scheme
+        if parsed.scheme not in ('ws', 'wss'):
+            raise ValueError("URL must use 'ws://' or 'wss://' scheme")
+
+        # Validate hostname and port
+        if not parsed.netloc:
+            raise ValueError("Invalid hostname")
+
+        # Ensure path starts with /
+        path = parsed.path
+        if not path.startswith('/'):
+            path = '/' + path
+
+        # Reconstruct the base URL with validated parts
+        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
+
+        # Add query parameters
+        if query_params:
+            query_string = urlencode(query_params)
+            return f"{validated_base}?{query_string}"
+
+        return validated_base
+
+    except Exception as e:
+        raise ValueError(f"Invalid WebSocket URL: {str(e)}")
+
+
+def js_bypass_path(filename):
+    current_directory = os.path.dirname(__file__)
+    return os.path.join(current_directory, 'bypasses', filename)
+
+
+def get_os_name():
+    # Get the OS name in the same format needed for browserforge
+    os_name = platform.system()
+    return {
+        'Linux': 'linux',
+        'Darwin': 'macos',
+        'Windows': 'windows',
+        # For the future? because why not
+        'iOS': 'ios',
+    }.get(os_name)
+
+
+def generate_suitable_fingerprint():
+    # This would be for Browserforge playwright injector
+    os_name = get_os_name()
+    return FingerprintGenerator(
+        browser=[Browser(name='chrome', min_version=128)],
+        os=os_name,  # None is ignored
+        device='desktop'
+    ).generate()
+
+
+def generate_headers(browser_mode=False):
+    if browser_mode:
+        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
+        # So we don't raise any inconsistency red flags while websites fingerprinting us
+        os_name = get_os_name()
+        return HeaderGenerator(
+            browser=[Browser(name='chrome', min_version=128)],
+            os=os_name,  # None is ignored
+            device='desktop'
+        ).generate()
+    else:
+        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
+        browsers = [
+            Browser(name='chrome', min_version=120),
+            Browser(name='firefox', min_version=120),
+            Browser(name='edge', min_version=120),
+        ]
+        return HeaderGenerator(browser=browsers, device='desktop').generate()
+
+
+def get_variable_name(var: Any) -> Optional[str]:
+    """Get the name of a variable using global and local scopes.
+    :param var: The variable to find the name for
+    :return: The name of the variable if found, None otherwise
+    """
+    for scope in [globals(), locals()]:
+        for name, value in scope.items():
+            if value is var:
+                return name
+    return None
+
+
+def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
+    """Check if a variable matches the specified type constraints.
+    :param variable: The variable to check
+    :param valid_types: List of valid types for the variable
+    :param default_value: Value to return if type check fails
+    :param critical: If True, raises TypeError instead of logging error
+    :param param_name: Optional parameter name for error messages
+    :return: The original variable if valid, default_value if invalid
+    :raise TypeError: If critical=True and type check fails
+    """
+    # Use provided param_name or try to get it automatically
+    var_name = param_name or get_variable_name(variable) or "Unknown"
+
+    # Convert valid_types to a list if None
+    valid_types = valid_types or []
+
+    # Handle None value
+    if variable is None:
+        if type(None) in valid_types:
+            return variable
+        error_msg = f'Argument "{var_name}" cannot be None'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    # If no valid_types specified and variable has a value, return it
+    if not valid_types:
+        return variable
+
+    # Check if variable type matches any of the valid types
+    if not any(isinstance(variable, t) for t in valid_types):
+        type_names = [t.__name__ for t in valid_types]
+        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    return variable

From aadae4a25b374f2dc642a37ecbf83d2ba55ba999 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:12:37 +0300
Subject: [PATCH 011/118] Adding the JavaScript bypasses files for stealth mode

---
 .../engines/bypasses/navigator_plugins.js     |  40 ++++
 .../bypasses/notification_permission.js       |   5 +
 scrapling/engines/bypasses/pdf_viewer.js      |   5 +
 .../bypasses/playwright_fingerprint.js        |   2 +
 scrapling/engines/bypasses/screen_props.js    |  27 +++
 scrapling/engines/bypasses/webdriver_fully.js |  27 +++
 scrapling/engines/bypasses/window_chrome.js   | 213 ++++++++++++++++++
 7 files changed, 319 insertions(+)
 create mode 100644 scrapling/engines/bypasses/navigator_plugins.js
 create mode 100644 scrapling/engines/bypasses/notification_permission.js
 create mode 100644 scrapling/engines/bypasses/pdf_viewer.js
 create mode 100644 scrapling/engines/bypasses/playwright_fingerprint.js
 create mode 100644 scrapling/engines/bypasses/screen_props.js
 create mode 100644 scrapling/engines/bypasses/webdriver_fully.js
 create mode 100644 scrapling/engines/bypasses/window_chrome.js

diff --git a/scrapling/engines/bypasses/navigator_plugins.js b/scrapling/engines/bypasses/navigator_plugins.js
new file mode 100644
index 0000000..653fa5f
--- /dev/null
+++ b/scrapling/engines/bypasses/navigator_plugins.js
@@ -0,0 +1,40 @@
+if(navigator.plugins.length == 0){
+    Object.defineProperty(navigator, 'plugins', {
+        get: () => {
+            const PDFViewerPlugin = Object.create(Plugin.prototype, {
+                description: { value: 'Portable Document Format', enumerable: false },
+                filename: { value: 'internal-pdf-viewer', enumerable: false },
+                name: { value: 'PDF Viewer', enumerable: false },
+            });
+            const ChromePDFViewer = Object.create(Plugin.prototype, {
+                description: { value: 'Portable Document Format', enumerable: false },
+                filename: { value: 'internal-pdf-viewer', enumerable: false },
+                name: { value: 'Chrome PDF Viewer', enumerable: false },
+            });
+            const ChromiumPDFViewer = Object.create(Plugin.prototype, {
+                description: { value: 'Portable Document Format', enumerable: false },
+                filename: { value: 'internal-pdf-viewer', enumerable: false },
+                name: { value: 'Chromium PDF Viewer', enumerable: false },
+            });
+            const EdgePDFViewer = Object.create(Plugin.prototype, {
+                description: { value: 'Portable Document Format', enumerable: false },
+                filename: { value: 'internal-pdf-viewer', enumerable: false },
+                name: { value: 'Microsoft Edge PDF Viewer', enumerable: false },
+            });
+            const WebKitPDFPlugin = Object.create(Plugin.prototype, {
+                description: { value: 'Portable Document Format', enumerable: false },
+                filename: { value: 'internal-pdf-viewer', enumerable: false },
+                name: { value: 'WebKit built-in PDF', enumerable: false },
+            });
+
+            return Object.create(PluginArray.prototype, {
+                length: { value: 5 },
+                0: { value: PDFViewerPlugin },
+                1: { value: ChromePDFViewer },
+                2: { value: ChromiumPDFViewer },
+                3: { value: EdgePDFViewer },
+                4: { value: WebKitPDFPlugin },
+            });
+        },
+    });
+}
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/notification_permission.js b/scrapling/engines/bypasses/notification_permission.js
new file mode 100644
index 0000000..0c9c676
--- /dev/null
+++ b/scrapling/engines/bypasses/notification_permission.js
@@ -0,0 +1,5 @@
+// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections
+const isSecure = document.location.protocol.startsWith('https')
+if (isSecure){
+    Object.defineProperty(Notification, 'permission', {get: () => 'default'})
+}
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/pdf_viewer.js b/scrapling/engines/bypasses/pdf_viewer.js
new file mode 100644
index 0000000..4c88702
--- /dev/null
+++ b/scrapling/engines/bypasses/pdf_viewer.js
@@ -0,0 +1,5 @@
+// PDF viewer enabled
+// Bypasses `pdfIsDisabled` test in creepsjs's 'Like Headless' sections
+Object.defineProperty(navigator, 'pdfViewerEnabled', {
+    get: () => true,
+});
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/playwright_fingerprint.js b/scrapling/engines/bypasses/playwright_fingerprint.js
new file mode 100644
index 0000000..bfba6be
--- /dev/null
+++ b/scrapling/engines/bypasses/playwright_fingerprint.js
@@ -0,0 +1,2 @@
+// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915
+delete __pwInitScripts;
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/screen_props.js b/scrapling/engines/bypasses/screen_props.js
new file mode 100644
index 0000000..5056b4b
--- /dev/null
+++ b/scrapling/engines/bypasses/screen_props.js
@@ -0,0 +1,27 @@
+const windowScreenProps = {
+    // Dimensions
+    innerHeight: 0,
+    innerWidth: 0,
+    outerHeight: 754,
+    outerWidth: 1313,
+
+    // Position
+    screenX: 19,
+    pageXOffset: 0,
+    pageYOffset: 0,
+
+    // Display
+    devicePixelRatio: 2
+};
+
+try {
+    for (const [prop, value] of Object.entries(windowScreenProps)) {
+        if (value > 0) {
+            // The 0 values are introduced by collecting in the hidden iframe.
+            // They are document sizes anyway so no need to test them or inject them.
+            window[prop] = value;
+        }
+    }
+} catch (e) {
+    console.warn(e);
+};
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/webdriver_fully.js b/scrapling/engines/bypasses/webdriver_fully.js
new file mode 100644
index 0000000..4bda260
--- /dev/null
+++ b/scrapling/engines/bypasses/webdriver_fully.js
@@ -0,0 +1,27 @@
+// Create a function that looks like a native getter
+const nativeGetter = function get webdriver() {
+    return false;
+};
+
+// Copy over native function properties
+Object.defineProperties(nativeGetter, {
+    name: { value: 'get webdriver', configurable: true },
+    length: { value: 0, configurable: true },
+    toString: {
+        value: function() {
+            return `function get webdriver() { [native code] }`;
+        },
+        configurable: true
+    }
+});
+
+// Make it look native
+Object.setPrototypeOf(nativeGetter, Function.prototype);
+
+// Apply the modified descriptor
+Object.defineProperty(Navigator.prototype, 'webdriver', {
+    get: nativeGetter,
+    set: undefined,
+    enumerable: true,
+    configurable: true
+});
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/window_chrome.js b/scrapling/engines/bypasses/window_chrome.js
new file mode 100644
index 0000000..ba63a9c
--- /dev/null
+++ b/scrapling/engines/bypasses/window_chrome.js
@@ -0,0 +1,213 @@
+// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322
+// Faking window.chrome fully
+
+if (!window.chrome) {
+    // First, save all existing properties
+    const originalKeys = Object.getOwnPropertyNames(window);
+    const tempObj = {};
+
+    // Recreate all properties in original order
+    for (const key of originalKeys) {
+        const descriptor = Object.getOwnPropertyDescriptor(window, key);
+        const value = window[key];
+        // delete window[key];
+        Object.defineProperty(tempObj, key, descriptor);
+    }
+
+    // Use the exact property descriptor found in headful Chrome
+    // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
+    const mockChrome = {
+        loadTimes: {},
+        csi: {},
+        app: {
+            isInstalled: false
+        },
+        // Add other Chrome-specific properties
+    };
+
+    Object.defineProperty(tempObj, 'chrome', {
+        writable: true,
+        enumerable: true,
+        configurable: false,
+        value: mockChrome
+    });
+    for (const key of Object.getOwnPropertyNames(tempObj)) {
+        try {
+            Object.defineProperty(window, key,
+                Object.getOwnPropertyDescriptor(tempObj, key));
+        } catch (e) {}
+    };
+    // todo: solve this
+    // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121
+    // Chrome object have to be in the end of the window properties
+    // Object.assign(window, tempObj);
+    // But makes window.chrome unreadable on 'https://bot.sannysoft.com/'
+}
+
+// That means we're running headful and don't need to mock anything
+if ('app' in window.chrome) {
+    return; // Nothing to do here
+}
+const makeError = {
+    ErrorInInvocation: fn => {
+        const err = new TypeError(`Error in invocation of app.${fn}()`);
+        return utils.stripErrorWithAnchor(
+            err,
+            `at ${fn} (eval at <anonymous>`,
+        );
+    },
+};
+// check with: `JSON.stringify(window.chrome['app'])`
+const STATIC_DATA = JSON.parse(
+    `
+{
+  "isInstalled": false,
+  "InstallState": {
+    "DISABLED": "disabled",
+    "INSTALLED": "installed",
+    "NOT_INSTALLED": "not_installed"
+  },
+  "RunningState": {
+    "CANNOT_RUN": "cannot_run",
+    "READY_TO_RUN": "ready_to_run",
+    "RUNNING": "running"
+  }
+}
+    `.trim(),
+    );
+window.chrome.app = {
+    ...STATIC_DATA,
+
+    get isInstalled() {
+        return false;
+    },
+
+    getDetails: function getDetails() {
+        if (arguments.length) {
+            throw makeError.ErrorInInvocation(`getDetails`);
+        }
+        return null;
+    },
+    getIsInstalled: function getDetails() {
+        if (arguments.length) {
+            throw makeError.ErrorInInvocation(`getIsInstalled`);
+        }
+        return false;
+    },
+    runningState: function getDetails() {
+        if (arguments.length) {
+            throw makeError.ErrorInInvocation(`runningState`);
+        }
+        return 'cannot_run';
+    },
+};
+// Check that the Navigation Timing API v1 is available, we need that
+if (!window.performance || !window.performance.timing) {
+    return;
+}
+const {timing} = window.performance;
+window.chrome.csi = function () {
+    return {
+        onloadT: timing.domContentLoadedEventEnd,
+        startE: timing.navigationStart,
+        pageT: Date.now() - timing.navigationStart,
+        tran: 15, // Transition type or something
+    };
+};
+if (!window.PerformancePaintTiming){
+    return;
+}
+const {performance} = window;
+// Some stuff is not available on about:blank as it requires a navigation to occur,
+// let's harden the code to not fail then:
+const ntEntryFallback = {
+    nextHopProtocol: 'h2',
+    type: 'other',
+};
+
+// The API exposes some funky info regarding the connection
+const protocolInfo = {
+    get connectionInfo() {
+        const ntEntry =
+            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+        return ntEntry.nextHopProtocol;
+    },
+    get npnNegotiatedProtocol() {
+        // NPN is deprecated in favor of ALPN, but this implementation returns the
+        // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
+        const ntEntry =
+            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
+            ? ntEntry.nextHopProtocol
+            : 'unknown';
+    },
+    get navigationType() {
+        const ntEntry =
+            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+        return ntEntry.type;
+    },
+    get wasAlternateProtocolAvailable() {
+        // The Alternate-Protocol header is deprecated in favor of Alt-Svc
+        // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this
+        // should always return false.
+        return false;
+    },
+    get wasFetchedViaSpdy() {
+        // SPDY is deprecated in favor of HTTP/2, but this implementation returns
+        // true for HTTP/2 or HTTP2+QUIC/39 as well.
+        const ntEntry =
+            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
+    },
+    get wasNpnNegotiated() {
+        // NPN is deprecated in favor of ALPN, but this implementation returns true
+        // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
+        const ntEntry =
+            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
+    },
+};
+
+// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3
+function toFixed(num, fixed) {
+    var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?');
+    return num.toString().match(re)[0];
+}
+
+const timingInfo = {
+    get firstPaintAfterLoadTime() {
+        // This was never actually implemented and always returns 0.
+        return 0;
+    },
+    get requestTime() {
+        return timing.navigationStart / 1000;
+    },
+    get startLoadTime() {
+        return timing.navigationStart / 1000;
+    },
+    get commitLoadTime() {
+        return timing.responseStart / 1000;
+    },
+    get finishDocumentLoadTime() {
+        return timing.domContentLoadedEventEnd / 1000;
+    },
+    get finishLoadTime() {
+        return timing.loadEventEnd / 1000;
+    },
+    get firstPaintTime() {
+        const fpEntry = performance.getEntriesByType('paint')[0] || {
+            startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`)
+        };
+        return toFixed(
+            (fpEntry.startTime + performance.timeOrigin) / 1000,
+            3,
+        );
+    },
+};
+
+window.chrome.loadTimes = function () {
+    return {
+        ...protocolInfo,
+        ...timingInfo,
+    };
+};
\ No newline at end of file

From 3cb1696d6c8a5d97ea4765f403057d7f602d07cb Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:13:35 +0300
Subject: [PATCH 012/118] Adding the main fetcher file to use for fetching
 pages

---
 scrapling/fetcher.py | 65 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 scrapling/fetcher.py

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
new file mode 100644
index 0000000..065c46c
--- /dev/null
+++ b/scrapling/fetcher.py
@@ -0,0 +1,65 @@
+from scrapling._types import Any, Dict, Optional, Union
+
+from scrapling.engines import CamoufoxEngine, StaticEngine, check_if_engine_usable
+from scrapling.parser import Adaptor, SQLiteStorageSystem
+
+
+class Fetcher:
+    def __init__(
+            self,
+            browser_engine: Optional[object] = None,
+            # Adaptor class parameters
+            response_encoding: str = "utf8",
+            huge_tree: bool = True,
+            keep_comments: Optional[bool] = False,
+            auto_match: Optional[bool] = False,
+            storage: Any = SQLiteStorageSystem,
+            storage_args: Optional[Dict] = None,
+            debug: Optional[bool] = True,
+    ):
+        if browser_engine is not None:
+            self.engine = check_if_engine_usable(browser_engine)
+        else:
+            self.engine = CamoufoxEngine()
+        # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
+        self.__encoding = response_encoding
+        self.__huge_tree = huge_tree
+        self.__keep_comments = keep_comments
+        self.__auto_match = auto_match
+        self.__storage = storage
+        self.__storage_args = storage_args
+        self.__debug = debug
+
+    def __generate_adaptor(self, url, html_content):
+        """To make the code less repetitive and manage return result from one function"""
+        return Adaptor(
+            text=html_content,
+            url=url,
+            encoding=self.__encoding,
+            huge_tree=self.__huge_tree,
+            keep_comments=self.__keep_comments,
+            auto_match=self.__auto_match,
+            storage=self.__storage,
+            storage_args=self.__storage_args,
+            debug=self.__debug,
+        )
+
+    def fetch(self, url: str) -> Adaptor:
+        html_content = self.engine.fetch(url)
+        return self.__generate_adaptor(url, html_content)
+
+    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
+        html_content = StaticEngine(follow_redirects, timeout).get(url, stealthy_headers, **kwargs)
+        return self.__generate_adaptor(url, html_content)
+
+    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
+        html_content = StaticEngine(follow_redirects, timeout).post(url, stealthy_headers, **kwargs)
+        return self.__generate_adaptor(url, html_content)
+
+    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
+        html_content = StaticEngine(follow_redirects, timeout).put(url, stealthy_headers, **kwargs)
+        return self.__generate_adaptor(url, html_content)
+
+    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
+        html_content = StaticEngine(follow_redirects, timeout).delete(url, stealthy_headers, **kwargs)
+        return self.__generate_adaptor(url, html_content)

From be2847ffb8b19308a27e54c46a74216bea8edd77 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:14:41 +0300
Subject: [PATCH 013/118] Bumping the library for version 0.2 for now and more
 adjustments

---
 scrapling/__init__.py |  5 +++--
 setup.cfg             |  4 ++--
 setup.py              | 16 +++++++++++-----
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 64a8d26..72cb0bb 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -1,10 +1,11 @@
 # Declare top-level shortcuts
+from scrapling.fetcher import Fetcher
 from scrapling.parser import Adaptor, Adaptors
 from scrapling.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.1.2"
+__version__ = "0.2"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
-__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler']
+__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler', 'Fetcher']
diff --git a/setup.cfg b/setup.cfg
index 4102c89..3c6197a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,8 +1,8 @@
 [metadata]
 name = scrapling
-version = 0.1.2
+version = 0.2
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is a powerful, flexible, adaptive, and high-performance web scraping library for Python.
 license = BSD
-home-page = https://github.com/D4Vinci/Scrapling
\ No newline at end of file
+home_page = https://github.com/D4Vinci/Scrapling
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 52e42c3..be46ec9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,4 @@
-from setuptools import setup
+from setuptools import setup, find_packages
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="scrapling",
-    version="0.1.2",
+    version="0.2",
     description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It 
     simplifies the process of extracting data from websites, even when they undergo structural changes, and offers 
     impressive speed improvements over many popular scraping tools.""",
@@ -15,7 +15,7 @@ setup(
     author="Karim Shoair",
     author_email="karim.shoair@pm.me",
     license="BSD",
-    packages=["scrapling",],
+    packages=find_packages(),
     zip_safe=False,
     package_dir={
         "scrapling": "scrapling",
@@ -32,16 +32,17 @@ setup(
         "Natural Language :: English",
         "Topic :: Internet :: WWW/HTTP",
         "Topic :: Text Processing :: Markup",
+        "Topic :: Internet :: WWW/HTTP :: Browsers",
         "Topic :: Text Processing :: Markup :: HTML",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Programming Language :: Python :: Implementation :: CPython",
         "Typing :: Typed",
     ],
@@ -53,8 +54,13 @@ setup(
         "w3lib",
         "orjson>=3",
         "tldextract",
+        'httpx[brotli,zstd]',
+        'playwright',
+        'rebrowser-playwright',
+        'camoufox',
+        'browserforge',
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     url="https://github.com/D4Vinci/Scrapling",
     project_urls={
         "Documentation": "https://github.com/D4Vinci/Scrapling/tree/main/docs",  # For now

From 088334b40b3985fc6ca2eb09f93bd7139e89738c Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:16:09 +0300
Subject: [PATCH 014/118] Reflecting the new changes in the ReadMe for testers

---
 README.md | 75 +++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 62 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 9c44ce3..2fe34e3 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# 🕷️ Scrapling: Lightning-Fast, Adaptive Web Scraping for Python
+# 🕷️ ScrapLing: Lightning-Fast, Adaptive Web Scraping for Python
 [![Tests](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg)](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [![PyPI version](https://badge.fury.io/py/Scrapling.svg)](https://badge.fury.io/py/Scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![PyPI Downloads](https://static.pepy.tech/badge/scrapling)](https://pepy.tech/project/scrapling)
 
 Dealing with failing web scrapers due to website changes? Meet Scrapling.
@@ -69,15 +69,6 @@ quote.path  # DOM path to element (List)
 ```
 To keep it simple, all methods can be chained on top of each other as long as you are chaining methods that return an element (It's called an `Adaptor` object) or a List of Adaptors (It's called `Adaptors` object)
 
-### Installation
-Scrapling is a breeze to get started with - We only require at least Python 3.7 to work and the rest of the requirements are installed automatically with the package.
-```bash
-# Using pip
-pip install scrapling
-
-# Or the latest from GitHub
-pip install git+https://github.com/D4Vinci/Scrapling.git@master
-```
 
 ## Performance
 
@@ -110,6 +101,55 @@ Scrapling can find elements with more methods and it returns full element `Adapt
 
 > All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
 
+## Installation
+Scrapling is a breeze to get started with - Starting from version 0.2, we require at least Python 3.8 to work.
+```bash
+# Using pip
+pip install scrapling
+
+# Or the latest from GitHub
+pip install git+https://github.com/D4Vinci/Scrapling.git@main
+```
+Then in the commandline download the browser with
+<details><summary>Windows OS</summary>
+
+```bash
+camoufox fetch
+```
+</details>
+<details><summary>MacOS</summary>
+
+```bash
+python3 -m camoufox fetch
+```
+</details>
+<details><summary>Linux</summary>
+
+```bash
+python -m camoufox fetch
+```
+On a fresh installation of Linux, you may also need the following Firefox dependencies:
+- Debian-based distros
+    ```bash
+    sudo apt install -y libgtk-3-0 libx11-xcb1 libasound2
+    ```
+- Arch-based distros
+    ```bash
+    sudo pacman -S gtk3 libx11 libxcb cairo libasound alsa-lib
+    ```
+</details>
+
+> You can head to the official [Camoufox documentation](https://camoufox.com/python/installation/#download-the-browser) for more info on installation
+
+Or if you are going to use the other browsers options, then install playwright browsers with:
+```commandline
+playwright install
+```
+and then update the user agents files with:
+```commandline
+python -m browserforge update
+```
+
 ## Advanced Features
 ### Smart Navigation
 ```python
@@ -272,7 +312,7 @@ if not element:  # One day website changes?
     element = page.css('#p1', auto_match=True)  # Still finds it!
 # the rest of the code...
 ```
-> How does the auto-matching work? Check the [FAQs](#FAQs) section for that and other possible issues while auto-matching.
+> How does the auto-matching work? Check the [FAQs](#-enlightening-questions-and-faqs) section for that and other possible issues while auto-matching.
 
 **Notes:**
 1. Passing the `auto_save` argument without setting `auto_match` to `True` while initializing the Adaptor object will only result in ignoring the `auto_save` argument value and the following warning message
@@ -371,7 +411,7 @@ There are a lot of deep details skipped here to make this as short as possible s
 Note that implementing your storage system can be complex as there are some strict rules such as inheriting from the same abstract class, following the singleton design pattern used in other classes, and more. So make sure to read the docs first.
 
 
-## FAQs
+## ⚡ Enlightening Questions and FAQs
 This section addresses common questions about Scrapling, please read this section before opening an issue.
 
 ### How does auto-matching work?
@@ -423,6 +463,9 @@ Everybody is invited and welcome to contribute to Scrapling. There is a lot to d
 
 Please read the [contributing file](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before doing anything.
 
+## Disclaimer for Scrapling Project
+> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international laws regarding data scraping and privacy. The authors and contributors are not responsible for any misuse of this software. This library should not be used to violate the rights of others, for unethical purposes, or to use data in an unauthorized or illegal manner. Do not use it on any website unless you have permission from the website owner or within their allowed rules like `robots.txt` file, for example.
+
 ## License
 This work is licensed under BSD-3
 
@@ -430,8 +473,14 @@ This work is licensed under BSD-3
 This project includes code adapted from:
 - Parsel (BSD License) - Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/translator.py) submodule
 
+## Thanks and References
+- [brotector](https://github.com/kaliiiiiiiiii/brotector)
+- [fakebrowser](https://github.com/kkoooqq/fakebrowser)
+- [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches)
+
 ## Known Issues
 - In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone.
 - Currently, Scrapling is not compatible with async/await.
 
-<div align="center"><small>Made with ❤️ by Karim Shoair</small></div><br>
+---
+<div align="center"><small>Designed & crafted with ❤️ by Karim Shoair.</small></div><br>

From 78c718c1d2a4948fa07605ff14692ca03aba25cb Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 2 Nov 2024 21:17:56 +0200
Subject: [PATCH 015/118] I don't know how that happened!

---
 scrapling/_types.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapling/_types.py b/scrapling/_types.py
index ca6769c..f832320 100644
--- a/scrapling/_types.py
+++ b/scrapling/_types.py
@@ -7,13 +7,13 @@ from typing import (
 )
 
 try:
-    from scrapling._types import Protocol
+    from typing import Protocol
 except ImportError:
     # Added in Python 3.8
     Protocol = object
 
 try:
-    from scrapling._types import SupportsIndex
+    from typing import SupportsIndex
 except ImportError:
     # 'SupportsIndex' got added in Python 3.8
     SupportsIndex = None

From 7f1c06e7bfc9e6fe71fb2a911ddce8d82003dc4e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 2 Nov 2024 21:19:13 +0200
Subject: [PATCH 016/118] Pumping camoufox version up

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index be46ec9..e472dfd 100644
--- a/setup.py
+++ b/setup.py
@@ -57,7 +57,7 @@ setup(
         'httpx[brotli,zstd]',
         'playwright',
         'rebrowser-playwright',
-        'camoufox',
+        'camoufox>=0.3.6',
         'browserforge',
     ],
     python_requires=">=3.8",

From 145c03daffb8b7b3b2d25e78ee7d03f2e9e8d123 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 01:04:02 +0200
Subject: [PATCH 017/118] Big structure changes (check commit description)

- Moved most of the parser functions/files to the core package.
- Converted tools file to a package and made separate files for similar functions.
- Now all fetcher engines return a Response object
- Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing.
- I added a new custom fetcher so the user can create and use an engine.
- More...
---
 README.md                                  |   2 +
 scrapling/__init__.py                      |   6 +-
 scrapling/core/__init__.py                 |   0
 scrapling/{ => core}/_types.py             |   0
 scrapling/{ => core}/custom_types.py       |   4 +-
 scrapling/{ => core}/mixins.py             |   0
 scrapling/{ => core}/storage_adaptors.py   |   4 +-
 scrapling/{ => core}/translator.py         |   4 +-
 scrapling/{ => core}/utils.py              |   2 +-
 scrapling/engines/__init__.py              |   5 +-
 scrapling/engines/camo.py                  |  51 ++++--
 scrapling/engines/constants.py             | 108 +++++++++++++
 scrapling/engines/pw.py                    |  97 +++++-------
 scrapling/engines/static.py                |  29 +++-
 scrapling/engines/toolbelt/__init__.py     |  18 +++
 scrapling/engines/toolbelt/custom.py       | 136 ++++++++++++++++
 scrapling/engines/toolbelt/fingerprints.py |  65 ++++++++
 scrapling/engines/toolbelt/navigation.py   |  43 +++++
 scrapling/engines/tools.py                 | 174 ---------------------
 scrapling/fetcher.py                       | 134 +++++++++-------
 scrapling/parser.py                        |  12 +-
 21 files changed, 565 insertions(+), 329 deletions(-)
 create mode 100644 scrapling/core/__init__.py
 rename scrapling/{ => core}/_types.py (100%)
 rename scrapling/{ => core}/custom_types.py (98%)
 rename scrapling/{ => core}/mixins.py (100%)
 rename scrapling/{ => core}/storage_adaptors.py (98%)
 rename scrapling/{ => core}/translator.py (98%)
 rename scrapling/{ => core}/utils.py (98%)
 create mode 100644 scrapling/engines/constants.py
 create mode 100644 scrapling/engines/toolbelt/__init__.py
 create mode 100644 scrapling/engines/toolbelt/custom.py
 create mode 100644 scrapling/engines/toolbelt/fingerprints.py
 create mode 100644 scrapling/engines/toolbelt/navigation.py
 delete mode 100644 scrapling/engines/tools.py

diff --git a/README.md b/README.md
index 2fe34e3..5dcfd04 100644
--- a/README.md
+++ b/README.md
@@ -477,6 +477,8 @@ This project includes code adapted from:
 - [brotector](https://github.com/kaliiiiiiiiii/brotector)
 - [fakebrowser](https://github.com/kkoooqq/fakebrowser)
 - [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches)
+- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright)
+- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
 
 ## Known Issues
 - In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone.
diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 72cb0bb..8323d50 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -1,11 +1,11 @@
 # Declare top-level shortcuts
-from scrapling.fetcher import Fetcher
+from scrapling.fetcher import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
 from scrapling.parser import Adaptor, Adaptors
-from scrapling.custom_types import TextHandler, AttributesHandler
+from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
 __version__ = "0.2"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
-__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler', 'Fetcher']
+__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher']
diff --git a/scrapling/core/__init__.py b/scrapling/core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scrapling/_types.py b/scrapling/core/_types.py
similarity index 100%
rename from scrapling/_types.py
rename to scrapling/core/_types.py
diff --git a/scrapling/custom_types.py b/scrapling/core/custom_types.py
similarity index 98%
rename from scrapling/custom_types.py
rename to scrapling/core/custom_types.py
index 5ff3b5f..bffc36f 100644
--- a/scrapling/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -2,8 +2,8 @@ import re
 from types import MappingProxyType
 from collections.abc import Mapping
 
-from scrapling.utils import _is_iterable, flatten
-from scrapling._types import Dict, List, Union, Pattern
+from scrapling.core.utils import _is_iterable, flatten
+from scrapling.core._types import Dict, List, Union, Pattern
 
 from orjson import loads, dumps
 from w3lib.html import replace_entities as _replace_entities
diff --git a/scrapling/mixins.py b/scrapling/core/mixins.py
similarity index 100%
rename from scrapling/mixins.py
rename to scrapling/core/mixins.py
diff --git a/scrapling/storage_adaptors.py b/scrapling/core/storage_adaptors.py
similarity index 98%
rename from scrapling/storage_adaptors.py
rename to scrapling/core/storage_adaptors.py
index d9018a3..675b46d 100644
--- a/scrapling/storage_adaptors.py
+++ b/scrapling/core/storage_adaptors.py
@@ -5,8 +5,8 @@ import threading
 from hashlib import sha256
 from abc import ABC, abstractmethod
 
-from scrapling._types import Dict, Optional, Union
-from scrapling.utils import _StorageTools, cache
+from scrapling.core._types import Dict, Optional, Union
+from scrapling.core.utils import _StorageTools, cache
 
 from lxml import html
 from tldextract import extract as tld
diff --git a/scrapling/translator.py b/scrapling/core/translator.py
similarity index 98%
rename from scrapling/translator.py
rename to scrapling/core/translator.py
index 77679a6..41f5811 100644
--- a/scrapling/translator.py
+++ b/scrapling/core/translator.py
@@ -9,8 +9,8 @@ which will be important in future releases but most importantly...
 import re
 
 from w3lib.html import HTML5_WHITESPACE
-from scrapling.utils import cache
-from scrapling._types import Any, Optional, Protocol, Self
+from scrapling.core.utils import cache
+from scrapling.core._types import Any, Optional, Protocol, Self
 
 from cssselect.xpath import ExpressionError
 from cssselect.xpath import XPathExpr as OriginalXPathExpr
diff --git a/scrapling/utils.py b/scrapling/core/utils.py
similarity index 98%
rename from scrapling/utils.py
rename to scrapling/core/utils.py
index 5f8efc9..db5ef15 100644
--- a/scrapling/utils.py
+++ b/scrapling/core/utils.py
@@ -4,7 +4,7 @@ from itertools import chain
 # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
 from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
 
-from scrapling._types import Dict, Iterable, Any
+from scrapling.core._types import Dict, Iterable, Any
 
 from lxml import html
 
diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py
index 89b5d62..d91e20a 100644
--- a/scrapling/engines/__init__.py
+++ b/scrapling/engines/__init__.py
@@ -1,6 +1,7 @@
 from .camo import CamoufoxEngine
 from .static import StaticEngine
-from .pw import PlaywrightEngine, DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
-from .tools import check_if_engine_usable
+from .pw import PlaywrightEngine
+from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
+from .toolbelt import check_if_engine_usable
 
 __all__ = ['CamoufoxEngine', 'PlaywrightEngine']
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 26464ab..eebdf51 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -1,26 +1,28 @@
 import logging
-from scrapling._types import Union, Callable, Optional
+from scrapling.core._types import Union, Callable, Optional, Dict
 
-from .tools import check_type_validity, get_os_name, generate_convincing_referer
+from scrapling.engines.toolbelt import (
+    Response,
+    do_nothing,
+    get_os_name,
+    check_type_validity,
+    generate_convincing_referer,
+)
 
 from camoufox.sync_api import Camoufox
 
 
-def _do_nothing(page):
-    # Anything
-    return page
-
-
 class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, str] = True,
-            block_images: Optional[bool] = True,
+            block_images: Optional[bool] = False,
             block_webrtc: Optional[bool] = False,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
-            page_action: Callable = _do_nothing,
+            page_action: Callable = do_nothing,
             wait_selector: Optional[str] = None,
             wait_selector_state: str = 'attached',
+            adaptor_arguments: Dict = None
     ):
         self.headless = headless
         self.block_images = bool(block_images)
@@ -30,23 +32,24 @@ class CamoufoxEngine:
         if callable(page_action):
             self.page_action = page_action
         else:
-            self.page_action = _do_nothing
+            self.page_action = do_nothing
             logging.error('[Ignored] Argument "page_action" must be callable')
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
-    def fetch(self, url: str):
+    def fetch(self, url: str) -> Response:
         with Camoufox(
                 headless=self.headless,
-                block_images=self.block_images,
+                block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
                 os=get_os_name(),
                 block_webrtc=self.block_webrtc,
         ) as browser:
             page = browser.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
-            page.goto(url, referer=generate_convincing_referer(url))
+            res = page.goto(url, referer=generate_convincing_referer(url))
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
@@ -58,6 +61,24 @@ class CamoufoxEngine:
                 waiter = page.locator(self.wait_selector)
                 waiter.wait_for(state=self.wait_selector_state)
 
-            html = page.content()
+            content_type = res.headers.get('content-type', '')
+            # Parse charset from content-type
+            encoding = 'utf-8'  # default encoding
+            if 'charset=' in content_type.lower():
+                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+
+            response = Response(
+                url=res.url,
+                text=res.text(),
+                content=res.body(),
+                status=res.status,
+                reason=res.status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
+                headers=res.all_headers(),
+                request_headers=res.request.all_headers(),
+                adaptor_arguments=self.adaptor_arguments
+            )
             page.close()
-        return html
+
+        return response
diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py
new file mode 100644
index 0000000..245e5c0
--- /dev/null
+++ b/scrapling/engines/constants.py
@@ -0,0 +1,108 @@
+# Disable loading these resources for speed
+DEFAULT_DISABLED_RESOURCES = [
+    'font',
+    'image',
+    'media',
+    'beacon',
+    'object',
+    'imageset',
+    'texttrack',
+    'websocket',
+    'csp_report',
+    'stylesheet',
+]
+
+DEFAULT_STEALTH_FLAGS = [
+    # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
+    # Generally this will make the browser faster and less detectable
+    '--no-pings',
+    '--incognito',
+    '--test-type',
+    '--lang=en-US',
+    '--mute-audio',
+    '--no-first-run',
+    '--disable-sync',
+    '--hide-scrollbars',
+    '--disable-logging',
+    '--start-maximized',  # For headless check bypass
+    '--enable-async-dns',
+    '--disable-breakpad',
+    '--disable-infobars',
+    '--accept-lang=en-US',
+    '--use-mock-keychain',
+    '--disable-translate',
+    '--disable-extensions',
+    '--disable-voice-input',
+    '--window-position=0,0',
+    '--disable-wake-on-wifi',
+    '--ignore-gpu-blocklist',
+    '--enable-tcp-fast-open',
+    '--enable-web-bluetooth',
+    '--disable-hang-monitor',
+    '--password-store=basic',
+    '--disable-cloud-import',
+    '--disable-default-apps',
+    '--disable-print-preview',
+    '--disable-dev-shm-usage',
+    '--disable-popup-blocking',
+    '--metrics-recording-only',
+    '--disable-crash-reporter',
+    '--disable-partial-raster',
+    '--disable-gesture-typing',
+    '--disable-checker-imaging',
+    '--disable-prompt-on-repost',
+    '--force-color-profile=srgb',
+    '--font-render-hinting=none',
+    '--no-default-browser-check',
+    '--aggressive-cache-discard',
+    '--disable-component-update',
+    '--disable-cookie-encryption',
+    '--disable-domain-reliability',
+    '--disable-threaded-animation',
+    '--disable-threaded-scrolling',
+    # '--disable-reading-from-canvas',  # For Firefox
+    '--enable-simple-cache-backend',
+    '--disable-background-networking',
+    '--disable-session-crashed-bubble',
+    '--enable-surface-synchronization',
+    '--disable-image-animation-resync',
+    '--disable-renderer-backgrounding',
+    '--disable-ipc-flooding-protection',
+    '--prerender-from-omnibox=disabled',
+    '--safebrowsing-disable-auto-update',
+    '--disable-offer-upload-credit-cards',
+    '--disable-features=site-per-process',
+    '--disable-background-timer-throttling',
+    '--disable-new-content-rendering-timeout',
+    '--run-all-compositor-stages-before-draw',
+    '--disable-client-side-phishing-detection',
+    '--disable-backgrounding-occluded-windows',
+    '--disable-layer-tree-host-memory-pressure',
+    '--autoplay-policy=no-user-gesture-required',
+    '--disable-offer-store-unmasked-wallet-cards',
+    '--disable-blink-features=AutomationControlled',
+    '--webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--disable-component-extensions-with-background-pages',
+    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
+    '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
+    '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
+]
+
+# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
+NSTBROWSER_DEFAULT_QUERY = {
+    "once": True,
+    "headless": True,
+    "autoClose": True,
+    "fingerprint": {
+        "flags": {
+            "timezone": "BasedOnIp",
+            "screen": "Custom"
+        },
+        "platform": 'linux',  # support: windows, mac, linux
+        "kernel": 'chromium',  # only support: chromium
+        "kernelMilestone": '128',
+        "hardwareConcurrency": 8,
+        "deviceMemory": 8,
+    },
+}
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index db353fa..4d628c3 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -1,43 +1,17 @@
 import json
 import logging
-from scrapling._types import Union, Callable, Optional, List, Dict
+from scrapling.core._types import Union, Callable, Optional, List, Dict
 
-from .tools import check_type_validity, generate_headers, js_bypass_path, construct_websocket_url, generate_convincing_referer
-
-# Disable loading these resources for speed
-DEFAULT_DISABLED_RESOURCES = ['beacon', 'csp_report', 'font', 'image', 'imageset', 'media', 'object', 'texttrack', 'stylesheet', 'websocket']
-DEFAULT_STEALTH_FLAGS = [
-    # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
-    # Generally this will make the browser faster and less detectable
-    '--incognito', '--accept-lang=en-US', '--lang=en-US', '--no-pings', '--mute-audio', '--no-first-run', '--no-default-browser-check', '--disable-cloud-import',
-    '--disable-gesture-typing', '--disable-offer-store-unmasked-wallet-cards', '--disable-offer-upload-credit-cards', '--disable-print-preview', '--disable-voice-input',
-    '--disable-wake-on-wifi', '--disable-cookie-encryption', '--ignore-gpu-blocklist', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open',
-    '--prerender-from-omnibox=disabled', '--enable-web-bluetooth', '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
-    '--aggressive-cache-discard', '--disable-ipc-flooding-protection', '--disable-blink-features=AutomationControlled', '--test-type',
-    '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
-    '--disable-breakpad', '--disable-component-update', '--disable-domain-reliability', '--disable-sync', '--disable-client-side-phishing-detection',
-    '--disable-hang-monitor', '--disable-popup-blocking', '--disable-prompt-on-repost', '--metrics-recording-only', '--safebrowsing-disable-auto-update', '--password-store=basic',
-    '--autoplay-policy=no-user-gesture-required', '--use-mock-keychain', '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
-    '--webrtc-ip-handling-policy=disable_non_proxied_udp', '--disable-session-crashed-bubble', '--disable-crash-reporter', '--disable-dev-shm-usage', '--force-color-profile=srgb',
-    '--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-infobars',
-    '--hide-scrollbars', '--disable-renderer-backgrounding', '--font-render-hinting=none', '--disable-logging', '--enable-surface-synchronization',
-    '--run-all-compositor-stages-before-draw', '--disable-threaded-animation', '--disable-threaded-scrolling', '--disable-checker-imaging',
-    '--disable-new-content-rendering-timeout', '--disable-image-animation-resync', '--disable-partial-raster',
-    '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
-    '--disable-layer-tree-host-memory-pressure',
-    '--window-position=0,0',
-    '--disable-features=site-per-process',
-    '--disable-default-apps',
-    '--disable-component-extensions-with-background-pages',
-    '--disable-extensions',
-    # "--disable-reading-from-canvas",  # For Firefox
-    '--start-maximized'  # For headless check bypass
-]
-
-
-def _do_nothing(page):
-    # Anything
-    return page
+from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
+from scrapling.engines.toolbelt import (
+    Response,
+    do_nothing,
+    js_bypass_path,
+    generate_headers,
+    check_type_validity,
+    construct_websocket_url,
+    generate_convincing_referer,
+)
 
 
 class PlaywrightEngine:
@@ -47,7 +21,7 @@ class PlaywrightEngine:
             useragent: Optional[str] = None,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
-            page_action: Callable = _do_nothing,
+            page_action: Callable = do_nothing,
             wait_selector: Optional[str] = None,
             wait_selector_state: Optional[str] = 'attached',
             stealth: bool = False,
@@ -56,6 +30,7 @@ class PlaywrightEngine:
             cdp_url: Optional[str] = None,
             nstbrowser_mode: bool = False,
             nstbrowser_config: Optional[Dict] = None,
+            adaptor_arguments: Dict = None
     ):
         self.headless = headless
         self.disable_resources = disable_resources
@@ -69,13 +44,14 @@ class PlaywrightEngine:
         if callable(page_action):
             self.page_action = page_action
         else:
-            self.page_action = _do_nothing
+            self.page_action = do_nothing
             logging.error('[Ignored] Argument "page_action" must be callable')
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state
         self.nstbrowser_mode = bool(nstbrowser_mode)
         self.nstbrowser_config = nstbrowser_config
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
     def _cdp_url_logic(self, flags: Optional[dict] = None):
         cdp_url = self.cdp_url
@@ -83,23 +59,7 @@ class PlaywrightEngine:
             if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
                 config = self.nstbrowser_config
             else:
-                # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
-                query = {
-                    "once": True,
-                    "headless": True,
-                    "autoClose": True,
-                    "fingerprint": {
-                        "flags": {
-                            "timezone": "BasedOnIp",
-                            "screen": "Custom"
-                        },
-                        "platform": 'linux',  # support: windows, mac, linux
-                        "kernel": 'chromium',  # only support: chromium
-                        "kernelMilestone": '128',
-                        "hardwareConcurrency": 8,
-                        "deviceMemory": 8,
-                    },
-                }
+                query = NSTBROWSER_DEFAULT_QUERY.copy()
                 if flags:
                     query.update({
                         "args": dict(zip(flags, [''] * len(flags))),  # browser args should be a dictionary
@@ -113,7 +73,7 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    def fetch(self, url):
+    def fetch(self, url) -> Response:
         if not self.stealth:
             from playwright.sync_api import sync_playwright
         else:
@@ -192,7 +152,7 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
@@ -204,6 +164,23 @@ class PlaywrightEngine:
                 waiter = page.locator(self.wait_selector)
                 waiter.wait_for(state=self.wait_selector_state)
 
-            html = page.content()
+            content_type = res.headers.get('content-type', '')
+            # Parse charset from content-type
+            encoding = 'utf-8'  # default encoding
+            if 'charset=' in content_type.lower():
+                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+
+            response = Response(
+                url=res.url,
+                text=res.text(),
+                content=res.body(),
+                status=res.status,
+                reason=res.status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
+                headers=res.all_headers(),
+                request_headers=res.request.all_headers(),
+                adaptor_arguments=self.adaptor_arguments
+            )
             page.close()
-        return html
+        return response
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index d4110e7..273e14b 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -1,9 +1,10 @@
 import logging
-from scrapling._types import Union, Optional, Dict
 
-from .tools import generate_convincing_referer, generate_headers
+from scrapling.core._types import Union, Optional, Dict
+from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 import httpx
+from httpx._models import Response as httpxResponse
 
 
 class StaticEngine:
@@ -11,10 +12,12 @@ class StaticEngine:
             self,
             follow_redirects: bool = True,
             timeout: Optional[Union[int, float]] = None,
+            adaptor_arguments: Dict = None
     ):
         self.timeout = timeout
         self.follow_redirects = bool(follow_redirects)
         self._extra_headers = generate_headers(browser_mode=False)
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
     @staticmethod
     def _headers_job(headers, url, stealth):
@@ -32,22 +35,36 @@ class StaticEngine:
 
         return headers
 
+    def _prepare_response(self, response: httpxResponse):
+        return Response(
+            url=str(response.url),
+            text=response.text,
+            content=response.content,
+            status=response.status_code,
+            reason=response.reason_phrase,
+            encoding=response.encoding or 'utf-8',
+            cookies=dict(response.cookies),
+            headers=dict(response.headers),
+            request_headers=response.request.headers,
+            adaptor_arguments=self.adaptor_arguments
+        )
+
     def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-        return request.text
+        return self._prepare_response(request)
 
     def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-        return request.text
+        return self._prepare_response(request)
 
     def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-        return request.text
+        return self._prepare_response(request)
 
     def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-        return request.text
+        return self._prepare_response(request)
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
new file mode 100644
index 0000000..3aa0aed
--- /dev/null
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -0,0 +1,18 @@
+from .fingerprints import (
+    get_os_name,
+    generate_headers,
+    generate_convincing_referer,
+    generate_suitable_fingerprint,
+)
+from .custom import (
+    Response,
+    do_nothing,
+    BaseFetcher,
+    get_variable_name,
+    check_type_validity,
+    check_if_engine_usable,
+)
+from .navigation import (
+    js_bypass_path,
+    construct_websocket_url,
+)
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
new file mode 100644
index 0000000..990244e
--- /dev/null
+++ b/scrapling/engines/toolbelt/custom.py
@@ -0,0 +1,136 @@
+"""
+Functions related to custom types or type checking
+"""
+import inspect
+import logging
+from dataclasses import dataclass, field
+
+from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict
+
+
+@dataclass(frozen=True)
+class Response:
+    url: str
+    text: str
+    content: bytes
+    status: int
+    reason: str
+    encoding: str = 'utf-8'  # default encoding
+    cookies: Dict = field(default_factory=dict)
+    headers: Dict = field(default_factory=dict)
+    request_headers: Dict = field(default_factory=dict)
+    adaptor_arguments: Dict = field(default_factory=dict)
+
+    @property
+    def adaptor(self):
+        if self.text:
+            return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.content:
+            return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
+        return None
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
+
+
+class BaseFetcher:
+    def __init__(
+            self,
+            # Adaptor class parameters
+            huge_tree: bool = True,
+            keep_comments: Optional[bool] = False,
+            auto_match: Optional[bool] = False,
+            storage: Any = SQLiteStorageSystem,
+            storage_args: Optional[Dict] = None,
+            debug: Optional[bool] = True,
+    ):
+        # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
+        self.adaptor_arguments = dict(
+            huge_tree=huge_tree,
+            keep_comments=keep_comments,
+            auto_match=auto_match,
+            storage=storage,
+            storage_args=storage_args,
+            debug=debug,
+        )
+
+
+def check_if_engine_usable(engine):
+    # if isinstance(engine, type):
+    #     raise TypeError("Expected an engine instance, not a class definition of the engine")
+
+    if hasattr(engine, 'fetch'):
+        fetch_function = getattr(engine, "fetch")
+        if callable(fetch_function):
+            if len(inspect.signature(fetch_function).parameters) > 0:
+                return engine
+            else:
+                # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
+                raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
+        else:
+            # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
+            raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
+    else:
+        # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
+        raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
+
+
+def get_variable_name(var: Any) -> Optional[str]:
+    """Get the name of a variable using global and local scopes.
+    :param var: The variable to find the name for
+    :return: The name of the variable if found, None otherwise
+    """
+    for scope in [globals(), locals()]:
+        for name, value in scope.items():
+            if value is var:
+                return name
+    return None
+
+
+def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
+    """Check if a variable matches the specified type constraints.
+    :param variable: The variable to check
+    :param valid_types: List of valid types for the variable
+    :param default_value: Value to return if type check fails
+    :param critical: If True, raises TypeError instead of logging error
+    :param param_name: Optional parameter name for error messages
+    :return: The original variable if valid, default_value if invalid
+    :raise TypeError: If critical=True and type check fails
+    """
+    # Use provided param_name or try to get it automatically
+    var_name = param_name or get_variable_name(variable) or "Unknown"
+
+    # Convert valid_types to a list if None
+    valid_types = valid_types or []
+
+    # Handle None value
+    if variable is None:
+        if type(None) in valid_types:
+            return variable
+        error_msg = f'Argument "{var_name}" cannot be None'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    # If no valid_types specified and variable has a value, return it
+    if not valid_types:
+        return variable
+
+    # Check if variable type matches any of the valid types
+    if not any(isinstance(variable, t) for t in valid_types):
+        type_names = [t.__name__ for t in valid_types]
+        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    return variable
+
+
+# Pew Pew
+def do_nothing(page):
+    # Just works as a filler for `page_action` argument in browser engines
+    return page
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
new file mode 100644
index 0000000..3969b0a
--- /dev/null
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -0,0 +1,65 @@
+"""
+Functions related to generating headers and fingerprints generally
+"""
+
+import platform
+
+from tldextract import extract
+from browserforge.fingerprints import FingerprintGenerator
+from browserforge.headers import HeaderGenerator, Browser
+
+
+def generate_convincing_referer(url):
+    """
+    Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
+
+    >>> generate_convincing_referer('https://www.somewebsite.com/blah')
+    'https://www.google.com/search?q=somewebsite'
+
+    :param url: The URL you are about to fetch.
+    :return:
+    """
+    website_name = extract(url).domain
+    return f'https://www.google.com/search?q={website_name}'
+
+
+def get_os_name():
+    # Get the OS name in the same format needed for browserforge
+    os_name = platform.system()
+    return {
+        'Linux': 'linux',
+        'Darwin': 'macos',
+        'Windows': 'windows',
+        # For the future? because why not
+        'iOS': 'ios',
+    }.get(os_name)
+
+
+def generate_suitable_fingerprint():
+    # This would be for Browserforge playwright injector
+    os_name = get_os_name()
+    return FingerprintGenerator(
+        browser=[Browser(name='chrome', min_version=128)],
+        os=os_name,  # None is ignored
+        device='desktop'
+    ).generate()
+
+
+def generate_headers(browser_mode=False):
+    if browser_mode:
+        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
+        # So we don't raise any inconsistency red flags while websites fingerprinting us
+        os_name = get_os_name()
+        return HeaderGenerator(
+            browser=[Browser(name='chrome', min_version=128)],
+            os=os_name,  # None is ignored
+            device='desktop'
+        ).generate()
+    else:
+        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
+        browsers = [
+            Browser(name='chrome', min_version=120),
+            Browser(name='firefox', min_version=120),
+            Browser(name='edge', min_version=120),
+        ]
+        return HeaderGenerator(browser=browsers, device='desktop').generate()
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
new file mode 100644
index 0000000..bf99cb2
--- /dev/null
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -0,0 +1,43 @@
+"""
+Functions related to files and URLs
+"""
+
+import os
+from urllib.parse import urlparse, urlencode
+
+
+def construct_websocket_url(base_url, query_params):
+    # Validate the base URL structure
+    try:
+        parsed = urlparse(base_url)
+
+        # Check scheme
+        if parsed.scheme not in ('ws', 'wss'):
+            raise ValueError("URL must use 'ws://' or 'wss://' scheme")
+
+        # Validate hostname and port
+        if not parsed.netloc:
+            raise ValueError("Invalid hostname")
+
+        # Ensure path starts with /
+        path = parsed.path
+        if not path.startswith('/'):
+            path = '/' + path
+
+        # Reconstruct the base URL with validated parts
+        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
+
+        # Add query parameters
+        if query_params:
+            query_string = urlencode(query_params)
+            return f"{validated_base}?{query_string}"
+
+        return validated_base
+
+    except Exception as e:
+        raise ValueError(f"Invalid WebSocket URL: {str(e)}")
+
+
+def js_bypass_path(filename):
+    current_directory = os.path.dirname(__file__)
+    return os.path.join(current_directory, 'bypasses', filename)
diff --git a/scrapling/engines/tools.py b/scrapling/engines/tools.py
deleted file mode 100644
index 3d4564e..0000000
--- a/scrapling/engines/tools.py
+++ /dev/null
@@ -1,174 +0,0 @@
-import os
-import logging
-import inspect
-import platform
-from scrapling._types import Any, List, Type, Union, Optional
-from urllib.parse import urlparse, urlencode
-
-from tldextract import extract
-from browserforge.fingerprints import FingerprintGenerator
-from browserforge.headers import HeaderGenerator, Browser
-
-
-def generate_convincing_referer(url):
-    """
-    Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
-
-    >>> generate_convincing_referer('https://www.somewebsite.com/blah')
-    'https://www.google.com/search?q=somewebsite'
-
-    :param url: The URL you are about to fetch.
-    :return:
-    """
-    website_name = extract(url).domain
-    return f'https://www.google.com/search?q={website_name}'
-
-
-def check_if_engine_usable(engine):
-    if isinstance(engine, type):
-        raise TypeError("Expected an engine instance, not a class definition of the engine")
-
-    if hasattr(engine, 'fetch'):
-        fetch_function = getattr(engine, "fetch")
-        if callable(fetch_function):
-            if len(inspect.signature(fetch_function).parameters) > 0:
-                return engine
-            else:
-                raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
-        else:
-            raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
-    else:
-        raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
-
-
-def construct_websocket_url(base_url, query_params):
-    # Validate the base URL structure
-    try:
-        parsed = urlparse(base_url)
-
-        # Check scheme
-        if parsed.scheme not in ('ws', 'wss'):
-            raise ValueError("URL must use 'ws://' or 'wss://' scheme")
-
-        # Validate hostname and port
-        if not parsed.netloc:
-            raise ValueError("Invalid hostname")
-
-        # Ensure path starts with /
-        path = parsed.path
-        if not path.startswith('/'):
-            path = '/' + path
-
-        # Reconstruct the base URL with validated parts
-        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
-
-        # Add query parameters
-        if query_params:
-            query_string = urlencode(query_params)
-            return f"{validated_base}?{query_string}"
-
-        return validated_base
-
-    except Exception as e:
-        raise ValueError(f"Invalid WebSocket URL: {str(e)}")
-
-
-def js_bypass_path(filename):
-    current_directory = os.path.dirname(__file__)
-    return os.path.join(current_directory, 'bypasses', filename)
-
-
-def get_os_name():
-    # Get the OS name in the same format needed for browserforge
-    os_name = platform.system()
-    return {
-        'Linux': 'linux',
-        'Darwin': 'macos',
-        'Windows': 'windows',
-        # For the future? because why not
-        'iOS': 'ios',
-    }.get(os_name)
-
-
-def generate_suitable_fingerprint():
-    # This would be for Browserforge playwright injector
-    os_name = get_os_name()
-    return FingerprintGenerator(
-        browser=[Browser(name='chrome', min_version=128)],
-        os=os_name,  # None is ignored
-        device='desktop'
-    ).generate()
-
-
-def generate_headers(browser_mode=False):
-    if browser_mode:
-        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
-        # So we don't raise any inconsistency red flags while websites fingerprinting us
-        os_name = get_os_name()
-        return HeaderGenerator(
-            browser=[Browser(name='chrome', min_version=128)],
-            os=os_name,  # None is ignored
-            device='desktop'
-        ).generate()
-    else:
-        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
-        browsers = [
-            Browser(name='chrome', min_version=120),
-            Browser(name='firefox', min_version=120),
-            Browser(name='edge', min_version=120),
-        ]
-        return HeaderGenerator(browser=browsers, device='desktop').generate()
-
-
-def get_variable_name(var: Any) -> Optional[str]:
-    """Get the name of a variable using global and local scopes.
-    :param var: The variable to find the name for
-    :return: The name of the variable if found, None otherwise
-    """
-    for scope in [globals(), locals()]:
-        for name, value in scope.items():
-            if value is var:
-                return name
-    return None
-
-
-def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
-    """Check if a variable matches the specified type constraints.
-    :param variable: The variable to check
-    :param valid_types: List of valid types for the variable
-    :param default_value: Value to return if type check fails
-    :param critical: If True, raises TypeError instead of logging error
-    :param param_name: Optional parameter name for error messages
-    :return: The original variable if valid, default_value if invalid
-    :raise TypeError: If critical=True and type check fails
-    """
-    # Use provided param_name or try to get it automatically
-    var_name = param_name or get_variable_name(variable) or "Unknown"
-
-    # Convert valid_types to a list if None
-    valid_types = valid_types or []
-
-    # Handle None value
-    if variable is None:
-        if type(None) in valid_types:
-            return variable
-        error_msg = f'Argument "{var_name}" cannot be None'
-        if critical:
-            raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
-        return default_value
-
-    # If no valid_types specified and variable has a value, return it
-    if not valid_types:
-        return variable
-
-    # Check if variable type matches any of the valid types
-    if not any(isinstance(variable, t) for t in valid_types):
-        type_names = [t.__name__ for t in valid_types]
-        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
-        if critical:
-            raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
-        return default_value
-
-    return variable
diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 065c46c..55c59b2 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -1,65 +1,87 @@
-from scrapling._types import Any, Dict, Optional, Union
+from scrapling.core._types import Dict, Optional, Union, Callable, List
 
-from scrapling.engines import CamoufoxEngine, StaticEngine, check_if_engine_usable
-from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing
+from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable
 
 
-class Fetcher:
-    def __init__(
-            self,
-            browser_engine: Optional[object] = None,
-            # Adaptor class parameters
-            response_encoding: str = "utf8",
-            huge_tree: bool = True,
-            keep_comments: Optional[bool] = False,
-            auto_match: Optional[bool] = False,
-            storage: Any = SQLiteStorageSystem,
-            storage_args: Optional[Dict] = None,
-            debug: Optional[bool] = True,
-    ):
-        if browser_engine is not None:
-            self.engine = check_if_engine_usable(browser_engine)
-        else:
-            self.engine = CamoufoxEngine()
-        # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
-        self.__encoding = response_encoding
-        self.__huge_tree = huge_tree
-        self.__keep_comments = keep_comments
-        self.__auto_match = auto_match
-        self.__storage = storage
-        self.__storage_args = storage_args
-        self.__debug = debug
+class Fetcher(BaseFetcher):
+    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs)
+        return response_object
 
-    def __generate_adaptor(self, url, html_content):
-        """To make the code less repetitive and manage return result from one function"""
-        return Adaptor(
-            text=html_content,
-            url=url,
-            encoding=self.__encoding,
-            huge_tree=self.__huge_tree,
-            keep_comments=self.__keep_comments,
-            auto_match=self.__auto_match,
-            storage=self.__storage,
-            storage_args=self.__storage_args,
-            debug=self.__debug,
+    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs)
+        return response_object
+
+    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
+        return response_object
+
+    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs)
+        return response_object
+
+
+class StealthyFetcher(BaseFetcher):
+    def fetch(
+            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False,
+            network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
+            wait_selector_state: str = 'attached',
+    ) -> Response:
+        engine = CamoufoxEngine(
+            timeout=timeout,
+            headless=headless,
+            page_action=page_action,
+            block_images=block_images,
+            block_webrtc=block_webrtc,
+            network_idle=network_idle,
+            wait_selector=wait_selector,
+            wait_selector_state=wait_selector_state,
+            adaptor_arguments=self.adaptor_arguments,
         )
+        return engine.fetch(url)
 
-    def fetch(self, url: str) -> Adaptor:
-        html_content = self.engine.fetch(url)
-        return self.__generate_adaptor(url, html_content)
 
-    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
-        html_content = StaticEngine(follow_redirects, timeout).get(url, stealthy_headers, **kwargs)
-        return self.__generate_adaptor(url, html_content)
+class PlayWrightFetcher(BaseFetcher):
+    def fetch(
+            self,
+            url: str,
+            headless: Union[bool, str] = True,
+            disable_resources: Optional[List] = None,
+            useragent: Optional[str] = None,
+            network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000,
+            page_action: Callable = do_nothing,
+            wait_selector: Optional[str] = None,
+            wait_selector_state: Optional[str] = 'attached',
+            stealth: bool = False,
+            hide_canvas: bool = True,
+            disable_webgl: bool = False,
+            cdp_url: Optional[str] = None,
+            nstbrowser_mode: bool = False,
+            nstbrowser_config: Optional[Dict] = None,
+    ) -> Response:
+        engine = PlaywrightEngine(
+            timeout=timeout,
+            stealth=stealth,
+            cdp_url=cdp_url,
+            headless=headless,
+            useragent=useragent,
+            page_action=page_action,
+            hide_canvas=hide_canvas,
+            network_idle=network_idle,
+            wait_selector=wait_selector,
+            disable_webgl=disable_webgl,
+            nstbrowser_mode=nstbrowser_mode,
+            nstbrowser_config=nstbrowser_config,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+            adaptor_arguments=self.adaptor_arguments,
+        )
+        return engine.fetch(url)
 
-    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
-        html_content = StaticEngine(follow_redirects, timeout).post(url, stealthy_headers, **kwargs)
-        return self.__generate_adaptor(url, html_content)
 
-    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
-        html_content = StaticEngine(follow_redirects, timeout).put(url, stealthy_headers, **kwargs)
-        return self.__generate_adaptor(url, html_content)
-
-    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
-        html_content = StaticEngine(follow_redirects, timeout).delete(url, stealthy_headers, **kwargs)
-        return self.__generate_adaptor(url, html_content)
+class CustomFetcher(BaseFetcher):
+    def fetch(self, url: str, browser_engine, **kwargs) -> Response:
+        engine = check_if_engine_usable(browser_engine)(adaptor_arguments=self.adaptor_arguments, **kwargs)
+        return engine.fetch(url)
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 771d08a..2dd4f66 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,12 +1,12 @@
 import os
 from difflib import SequenceMatcher
 
-from scrapling.translator import HTMLTranslator
-from scrapling.mixins import SelectorsGeneration
-from scrapling.custom_types import TextHandler, AttributesHandler
-from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
-from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
-from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
+from scrapling.core.translator import HTMLTranslator
+from scrapling.core.mixins import SelectorsGeneration
+from scrapling.core.custom_types import TextHandler, AttributesHandler
+from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
+from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
 
 from lxml import etree, html
 from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors

From 26265327b5805fac8c975cec5edd2104b57607b5 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 12:24:52 +0200
Subject: [PATCH 018/118] Adding the option to update browserforge with
 camoufox

Since my PRs to Camoufox got accepted :smoking:
---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 5dcfd04..31958c7 100644
--- a/README.md
+++ b/README.md
@@ -110,23 +110,23 @@ pip install scrapling
 # Or the latest from GitHub
 pip install git+https://github.com/D4Vinci/Scrapling.git@main
 ```
-Then in the commandline download the browser with
+- For using the `StealthyFetcher`, go in the commandline and download the browser with
 <details><summary>Windows OS</summary>
 
 ```bash
-camoufox fetch
+camoufox fetch --browserforge
 ```
 </details>
 <details><summary>MacOS</summary>
 
 ```bash
-python3 -m camoufox fetch
+python3 -m camoufox fetch --browserforge
 ```
 </details>
 <details><summary>Linux</summary>
 
 ```bash
-python -m camoufox fetch
+python -m camoufox fetch --browserforge
 ```
 On a fresh installation of Linux, you may also need the following Firefox dependencies:
 - Debian-based distros
@@ -139,13 +139,13 @@ On a fresh installation of Linux, you may also need the following Firefox depend
     ```
 </details>
 
-> You can head to the official [Camoufox documentation](https://camoufox.com/python/installation/#download-the-browser) for more info on installation
+<small> See the official <a href="https://camoufox.com/python/installation/#download-the-browser">Camoufox documentation</a> for more info on installation</small>
 
-Or if you are going to use the other browsers options, then install playwright browsers with:
+- If you are going to use the `PlayWrightFetcher` options, then install playwright browsers with:
 ```commandline
 playwright install
 ```
-and then update the user agents files with:
+- If you are going to use normal requests only with `Fetcher` class then update the fingerprints files with:
 ```commandline
 python -m browserforge update
 ```

From 263236a302bfc65428aaa30868c1019b5763e7bf Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 12:42:26 +0200
Subject: [PATCH 019/118] Update README.md

---
 README.md | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 31958c7..95697ed 100644
--- a/README.md
+++ b/README.md
@@ -104,11 +104,7 @@ Scrapling can find elements with more methods and it returns full element `Adapt
 ## Installation
 Scrapling is a breeze to get started with - Starting from version 0.2, we require at least Python 3.8 to work.
 ```bash
-# Using pip
-pip install scrapling
-
-# Or the latest from GitHub
-pip install git+https://github.com/D4Vinci/Scrapling.git@main
+pip3 install scrapling
 ```
 - For using the `StealthyFetcher`, go in the commandline and download the browser with
 <details><summary>Windows OS</summary>

From d845cae8169fade52ea3cf154b4ae67daf48cfc2 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 12:42:53 +0200
Subject: [PATCH 020/118] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7c50908..2e3b3b2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -27,4 +27,9 @@ Also, consider setting `debug` to `True` while initializing the Adaptor object s
  - Fork Scrapling [git repository](https://github.com/D4Vinci/Scrapling).
  - Make your changes.
  - Ensure tests work.
- - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling.
\ No newline at end of file
+ - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling.
+
+### Installing the latest changes from the dev branch
+```commandline
+pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev
+```

From 942aa08c8502c1eaa8c611f349e9db3f6bfb9baf Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:00:23 +0200
Subject: [PATCH 021/118] Handling JSON responses better

---
 scrapling/parser.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 2dd4f66..57a7529 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -601,7 +601,10 @@ class Adaptor(SelectorsGeneration):
     # Operations on text functions
     def json(self) -> Dict:
         """Return json response if the response is jsonable otherwise throws error"""
-        return self.text.json()
+        if self.text:
+            return self.text.json()
+        else:
+            return self.get_all_text(strip=True).json()
 
     def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True) -> 'List[str]':
         """Apply the given regex to the current text and return a list of strings with the matches.

From afe53332b8e351c7f290839d5b725f78c29456a6 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:01:48 +0200
Subject: [PATCH 022/118] Convert the request headers to dictionary to unify
 response object shape across engines

---
 scrapling/engines/static.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 273e14b..35b575a 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -45,7 +45,7 @@ class StaticEngine:
             encoding=response.encoding or 'utf-8',
             cookies=dict(response.cookies),
             headers=dict(response.headers),
-            request_headers=response.request.headers,
+            request_headers=dict(response.request.headers),
             adaptor_arguments=self.adaptor_arguments
         )
 

From 1ff1c93bcf2dcccb421de768cbb1fffd19acbbd9 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:04:02 +0200
Subject: [PATCH 023/118] Give response's body bytes priority over response's
 text

---
 scrapling/engines/toolbelt/custom.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 990244e..3d71b4f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -24,10 +24,10 @@ class Response:
 
     @property
     def adaptor(self):
-        if self.text:
-            return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
-        elif self.content:
+        if self.content:
             return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.text:
+            return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
         return None
 
     def __repr__(self):

From a0022f7322e69eba0766777a4508b16dd1e61d16 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:20:00 +0200
Subject: [PATCH 024/118] Pumping up camoufox version

Now, all progress bars are hidden
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e472dfd..5cb769d 100644
--- a/setup.py
+++ b/setup.py
@@ -57,7 +57,7 @@ setup(
         'httpx[brotli,zstd]',
         'playwright',
         'rebrowser-playwright',
-        'camoufox>=0.3.6',
+        'camoufox>=0.3.7',
         'browserforge',
     ],
     python_requires=">=3.8",

From eaae5908c9f9157e6e0cfb461e95c8f482c1b032 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:33:24 +0200
Subject: [PATCH 025/118] Make auto_match enabled by default

---
 scrapling/parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 57a7529..2d0ebf1 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -27,7 +27,7 @@ class Adaptor(SelectorsGeneration):
             huge_tree: bool = True,
             root: Optional[html.HtmlElement] = None,
             keep_comments: Optional[bool] = False,
-            auto_match: Optional[bool] = False,
+            auto_match: Optional[bool] = True,
             storage: Any = SQLiteStorageSystem,
             storage_args: Optional[Dict] = None,
             debug: Optional[bool] = True,

From 55ab1d63b7c69da02fe555f1f616acb8e8b43e48 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:37:27 +0200
Subject: [PATCH 026/118] Turn auto_match off when not needed in testing

---
 tests/test_parser_functions.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_parser_functions.py b/tests/test_parser_functions.py
index 80c8f0f..88013ad 100644
--- a/tests/test_parser_functions.py
+++ b/tests/test_parser_functions.py
@@ -127,16 +127,16 @@ class TestParser(unittest.TestCase):
     def test_expected_errors(self):
         """Test errors that should raised if it does"""
         with self.assertRaises(ValueError):
-            _ = Adaptor()
+            _ = Adaptor(auto_match=False)
 
         with self.assertRaises(TypeError):
-            _ = Adaptor(root="ayo")
+            _ = Adaptor(root="ayo", auto_match=False)
 
         with self.assertRaises(TypeError):
-            _ = Adaptor(text=1)
+            _ = Adaptor(text=1, auto_match=False)
 
         with self.assertRaises(TypeError):
-            _ = Adaptor(body=1)
+            _ = Adaptor(body=1, auto_match=False)
 
         with self.assertRaises(ValueError):
             _ = Adaptor(self.html, storage=object, auto_match=True)

From be350f21dd8e7b5f2b9fa68f04f323e79d862709 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 13:45:13 +0200
Subject: [PATCH 027/118] Adding documentation to the base fetcher...

+ Configure logger from here.
+ Enable auto_match by default.
---
 scrapling/engines/toolbelt/custom.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 3d71b4f..c06d68d 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -6,6 +6,7 @@ import logging
 from dataclasses import dataclass, field
 
 from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core.utils import setup_basic_logging
 from scrapling.core._types import Any, List, Type, Union, Optional, Dict
 
 
@@ -36,15 +37,23 @@ class Response:
 
 class BaseFetcher:
     def __init__(
-            self,
-            # Adaptor class parameters
-            huge_tree: bool = True,
-            keep_comments: Optional[bool] = False,
-            auto_match: Optional[bool] = False,
-            storage: Any = SQLiteStorageSystem,
-            storage_args: Optional[Dict] = None,
-            debug: Optional[bool] = True,
+            self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
+            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
     ):
+        """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
+        are detected and passed automatically from the Fetcher based on the response for accessibility.
+
+        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
+            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
+        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
+            priority over all auto-match related arguments/functions in the class.
+        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
+        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
+            If empty, default values will be used.
+        :param debug: Enable debug mode
+        """
+        # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
         self.adaptor_arguments = dict(
             huge_tree=huge_tree,
@@ -54,6 +63,8 @@ class BaseFetcher:
             storage_args=storage_args,
             debug=debug,
         )
+        # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
+        setup_basic_logging(level='debug' if debug else 'info')
 
 
 def check_if_engine_usable(engine):

From 2528d4e023eb4af542e73d2106a439bec33cd205 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 14:39:24 +0200
Subject: [PATCH 028/118] Add documentation for the Fetcher class

---
 scrapling/fetcher.py | 44 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 4 deletions(-)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 55c59b2..687cadc 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -5,19 +5,55 @@ from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, ch
 
 
 class Fetcher(BaseFetcher):
-    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx.
+
+    Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
+    """
+    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP GET request for you but with some added flavors.
+        :param url: Target url.
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs)
         return response_object
 
-    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP POST request for you but with some added flavors.
+        :param url: Target url.
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs)
         return response_object
 
-    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP PUT request for you but with some added flavors.
+        :param url: Target url
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
         return response_object
 
-    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP DELETE request for you but with some added flavors.
+        :param url: Target url
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs)
         return response_object
 

From e7300ecba0dce652a7754a7012f06cfc7cf68175 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 14:40:19 +0200
Subject: [PATCH 029/118] StealthFetcher - Add the option to allow webgl in the
 browser

---
 scrapling/engines/camo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index eebdf51..9e20914 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -17,6 +17,7 @@ class CamoufoxEngine:
             self, headless: Union[bool, str] = True,
             block_images: Optional[bool] = False,
             block_webrtc: Optional[bool] = False,
+            allow_webgl: Optional[bool] = False,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
             page_action: Callable = do_nothing,
@@ -27,6 +28,7 @@ class CamoufoxEngine:
         self.headless = headless
         self.block_images = bool(block_images)
         self.block_webrtc = bool(block_webrtc)
+        self.allow_webgl = bool(allow_webgl)
         self.network_idle = bool(network_idle)
         self.timeout = check_type_validity(timeout, [int, float], 30000)
         if callable(page_action):
@@ -45,6 +47,7 @@ class CamoufoxEngine:
                 block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
                 os=get_os_name(),
                 block_webrtc=self.block_webrtc,
+                allow_webgl=self.allow_webgl,
         ) as browser:
             page = browser.new_page()
             page.set_default_navigation_timeout(self.timeout)

From e7e499d64b455112cc82f71ad837e8ce6dadefd5 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 14:45:20 +0200
Subject: [PATCH 030/118] Add documentation for StealthyFetcher class + add
 webgl option

---
 scrapling/fetcher.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 687cadc..2b2369c 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -59,17 +59,38 @@ class Fetcher(BaseFetcher):
 
 
 class StealthyFetcher(BaseFetcher):
+    """A `Fetcher` class type that is completely stealthy fetcher that uses a modified version of Firefox.
+
+     It works as real browsers passing almost all online tests/protections based on Camoufox.
+    """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False,
+            allow_webgl: Optional[bool] = False,
             network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
             wait_selector_state: str = 'attached',
     ) -> Response:
+        """
+        Opens up a browser and do your request based on your chosen options below.
+        :param url: Target url.
+        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
+        :param network_idle: Wait for the page to not do do any requests.
+        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
+        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         engine = CamoufoxEngine(
             timeout=timeout,
             headless=headless,
             page_action=page_action,
             block_images=block_images,
             block_webrtc=block_webrtc,
+            allow_webgl=allow_webgl,
             network_idle=network_idle,
             wait_selector=wait_selector,
             wait_selector_state=wait_selector_state,

From 0867d9e78a74a003c0220e86b4b6921a761b6357 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 14:48:33 +0200
Subject: [PATCH 031/118] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 95697ed..8ac4e06 100644
--- a/README.md
+++ b/README.md
@@ -137,9 +137,9 @@ On a fresh installation of Linux, you may also need the following Firefox depend
 
 <small> See the official <a href="https://camoufox.com/python/installation/#download-the-browser">Camoufox documentation</a> for more info on installation</small>
 
-- If you are going to use the `PlayWrightFetcher` options, then install playwright browsers with:
+- If you are going to use the `PlayWrightFetcher` options, then install playwright's chromium browser with:
 ```commandline
-playwright install
+playwright install chromium
 ```
 - If you are going to use normal requests only with `Fetcher` class then update the fingerprints files with:
 ```commandline

From c8c494e73f09ff44bf00b4c0412ce07eb218e482 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:25:55 +0200
Subject: [PATCH 032/118] Adding a function to intercept and drop unwanted
 requests

---
 scrapling/engines/toolbelt/__init__.py   |  1 +
 scrapling/engines/toolbelt/navigation.py | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 3aa0aed..375e304 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -14,5 +14,6 @@ from .custom import (
 )
 from .navigation import (
     js_bypass_path,
+    intercept_route,
     construct_websocket_url,
 )
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index bf99cb2..5a8192b 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -3,7 +3,18 @@ Functions related to files and URLs
 """
 
 import os
+import logging
 from urllib.parse import urlparse, urlencode
+from playwright.sync_api import Route
+
+from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+
+
+def intercept_route(route: Route):
+    if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
+        logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        return route.abort()
+    return route.continue_()
 
 
 def construct_websocket_url(base_url, query_params):

From b32e1203f21f31d5eb172dd7ded1a39567723d98 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:27:32 +0200
Subject: [PATCH 033/118] StealthyFetcher - The option to drop unnecessary
 resources requests for speed boost

---
 scrapling/engines/camo.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 9e20914..b2cec20 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -5,6 +5,7 @@ from scrapling.engines.toolbelt import (
     Response,
     do_nothing,
     get_os_name,
+    intercept_route,
     check_type_validity,
     generate_convincing_referer,
 )
@@ -16,6 +17,7 @@ class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, str] = True,
             block_images: Optional[bool] = False,
+            disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False,
             allow_webgl: Optional[bool] = False,
             network_idle: Optional[bool] = False,
@@ -27,6 +29,7 @@ class CamoufoxEngine:
     ):
         self.headless = headless
         self.block_images = bool(block_images)
+        self.disable_resources = bool(disable_resources)
         self.block_webrtc = bool(block_webrtc)
         self.allow_webgl = bool(allow_webgl)
         self.network_idle = bool(network_idle)
@@ -52,6 +55,9 @@ class CamoufoxEngine:
             page = browser.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
+            if self.disable_resources:
+                page.route("**/*", intercept_route)
+
             res = page.goto(url, referer=generate_convincing_referer(url))
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")

From 711de8110637cce88d7832b2f1b5bbf526dcb1a0 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:27:55 +0200
Subject: [PATCH 034/118] PlaywrightFetcher - The option to drop unnecessary
 resources requests for speed boost

---
 scrapling/engines/pw.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 4d628c3..4ce0600 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
     Response,
     do_nothing,
     js_bypass_path,
+    intercept_route,
     generate_headers,
     check_type_validity,
     construct_websocket_url,
@@ -17,7 +18,7 @@ from scrapling.engines.toolbelt import (
 class PlaywrightEngine:
     def __init__(
             self, headless: Union[bool, str] = True,
-            disable_resources: Optional[List] = None,
+            disable_resources: Optional[bool] = False,
             useragent: Optional[str] = None,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
@@ -134,6 +135,9 @@ class PlaywrightEngine:
             page = context.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
+            if self.disable_resources:
+                page.route("**/*", intercept_route)
+
             if self.stealth:
                 # Basic bypasses nothing fancy as I'm still working on it
                 # But with adding these bypasses to the above config, it bypasses many online tests like

From 3556d11fb2f9a8a23c9f6fbb5c4db5e97d408a49 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:33:08 +0200
Subject: [PATCH 035/118] StealthyFetcher - Add `disable_resources` option and
 its explanation

---
 scrapling/fetcher.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 2b2369c..6a7e035 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -64,9 +64,9 @@ class StealthyFetcher(BaseFetcher):
      It works as real browsers passing almost all online tests/protections based on Camoufox.
     """
     def fetch(
-            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False,
-            allow_webgl: Optional[bool] = False,
-            network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
+            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = True,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
             wait_selector_state: str = 'attached',
     ) -> Response:
         """
@@ -74,7 +74,10 @@ class StealthyFetcher(BaseFetcher):
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
-            This can help save your proxy usage but careful with this option as it makes some websites never finish loading.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests to unnecessary resources for speed boost.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
         :param network_idle: Wait for the page to not do do any requests.
@@ -91,6 +94,7 @@ class StealthyFetcher(BaseFetcher):
             block_images=block_images,
             block_webrtc=block_webrtc,
             allow_webgl=allow_webgl,
+            disable_resources=disable_resources,
             network_idle=network_idle,
             wait_selector=wait_selector,
             wait_selector_state=wait_selector_state,

From 1d2c68125acf7c53acf2ceb96bf531dad98f4151 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:40:42 +0200
Subject: [PATCH 036/118] Update fetcher.py

---
 scrapling/fetcher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 6a7e035..6bed2fc 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -64,7 +64,7 @@ class StealthyFetcher(BaseFetcher):
      It works as real browsers passing almost all online tests/protections based on Camoufox.
     """
     def fetch(
-            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = True,
+            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
             wait_selector_state: str = 'attached',

From c1575b8e2e06f7a778fe6172c428bcdff5c2e67e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 16:49:33 +0200
Subject: [PATCH 037/118] Updating doc-strings for more clarity

---
 scrapling/fetcher.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index 6bed2fc..b00f6c9 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -14,7 +14,8 @@ class Fetcher(BaseFetcher):
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
@@ -26,7 +27,8 @@ class Fetcher(BaseFetcher):
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
@@ -38,7 +40,8 @@ class Fetcher(BaseFetcher):
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+        create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
@@ -50,7 +53,8 @@ class Fetcher(BaseFetcher):
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
@@ -62,6 +66,7 @@ class StealthyFetcher(BaseFetcher):
     """A `Fetcher` class type that is completely stealthy fetcher that uses a modified version of Firefox.
 
      It works as real browsers passing almost all online tests/protections based on Camoufox.
+     Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
     """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,

From 61fd592a6aeee349084fd721ad0714d7cd243605 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 21:04:11 +0200
Subject: [PATCH 038/118] Adding documentation to PlaywrightFetcher class

---
 scrapling/fetcher.py | 56 ++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 15 deletions(-)

diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py
index b00f6c9..80b913a 100644
--- a/scrapling/fetcher.py
+++ b/scrapling/fetcher.py
@@ -80,7 +80,7 @@ class StealthyFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
-        :param disable_resources: Drop requests to unnecessary resources for speed boost.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
@@ -109,24 +109,50 @@ class StealthyFetcher(BaseFetcher):
 
 
 class PlayWrightFetcher(BaseFetcher):
+    """A `Fetcher` class type that provide many options, all of them are based on PlayWright.
+
+     Using this Fetcher class, you can do requests with:
+        - Vanilla Playwright without any modifications other than the ones you chose.
+        - Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com
+        Some of the things stealth mode do includes:
+            1) Patches the CDP runtime fingerprint.
+            2) Mimics some of real browsers' properties by injects several JS files and using custom options.
+            3) Using custom flags on launch to hide playwright even more and make it faster.
+            4) Sets the referer of every request as if this request came from Google's search of this URL's domain.
+            5) Generates real browser's headers of the same type and same user OS then append it to the request.
+        - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+        - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
+        > Note that these are the main options with PlayWright but it can be mixed together.
+    """
     def fetch(
-            self,
-            url: str,
-            headless: Union[bool, str] = True,
-            disable_resources: Optional[List] = None,
-            useragent: Optional[str] = None,
-            network_idle: Optional[bool] = False,
-            timeout: Optional[float] = 30000,
-            page_action: Callable = do_nothing,
-            wait_selector: Optional[str] = None,
-            wait_selector_state: Optional[str] = 'attached',
+            self, url: str, headless: Union[bool, str] = True, disable_resources: Optional[List] = None,
+            useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
+            page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
+            hide_canvas: bool = True, disable_webgl: bool = False,
             stealth: bool = False,
-            hide_canvas: bool = True,
-            disable_webgl: bool = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: bool = False,
-            nstbrowser_config: Optional[Dict] = None,
+            nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
+        """Opens up a browser and do your request based on your chosen options below.
+        :param url: Target url.
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param network_idle: Wait for the page to not do do any requests.
+        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
+        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
+        :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         engine = PlaywrightEngine(
             timeout=timeout,
             stealth=stealth,

From 54cacb8a4e841baf4fa2202fbc54281f27dd49d3 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:20:29 +0200
Subject: [PATCH 039/118] Camoufox Engine - Doc strings and type annotations
 stuff

---
 scrapling/engines/camo.py | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index b2cec20..1514424 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -15,18 +15,28 @@ from camoufox.sync_api import Camoufox
 
 class CamoufoxEngine:
     def __init__(
-            self, headless: Union[bool, str] = True,
-            block_images: Optional[bool] = False,
-            disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False,
-            allow_webgl: Optional[bool] = False,
-            network_idle: Optional[bool] = False,
-            timeout: Optional[float] = 30000,
-            page_action: Callable = do_nothing,
-            wait_selector: Optional[str] = None,
-            wait_selector_state: str = 'attached',
-            adaptor_arguments: Dict = None
+            self, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
+            wait_selector_state: str = 'attached', adaptor_arguments: Dict = None
     ):
+        """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
+
+        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
+        :param network_idle: Wait for the page to not do do any requests.
+        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
+        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
         self.headless = headless
         self.block_images = bool(block_images)
         self.disable_resources = bool(disable_resources)
@@ -45,6 +55,11 @@ class CamoufoxEngine:
         self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
     def fetch(self, url: str) -> Response:
+        """Opens up the browser and do your request based on your chosen options.
+
+        :param url: Target url.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         with Camoufox(
                 headless=self.headless,
                 block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful

From 3af174526f8eb29626e2bb8f93eb767b86e3dec9 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:24:44 +0200
Subject: [PATCH 040/118] PlayWright Engine - Doc strings and type annotations
 stuff

---
 scrapling/engines/pw.py | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 4ce0600..aa2e0c2 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -33,6 +33,26 @@ class PlaywrightEngine:
             nstbrowser_config: Optional[Dict] = None,
             adaptor_arguments: Dict = None
     ):
+        """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
+
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param network_idle: Wait for the page to not do do any requests.
+        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
+        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
+        :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
         self.headless = headless
         self.disable_resources = disable_resources
         self.network_idle = bool(network_idle)
@@ -54,7 +74,12 @@ class PlaywrightEngine:
         self.nstbrowser_config = nstbrowser_config
         self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
-    def _cdp_url_logic(self, flags: Optional[dict] = None):
+    def _cdp_url_logic(self, flags: Optional[List] = None) -> str:
+        """Constructs new CDP URL if NSTBrowser is enabled otherwise return CDP URL as it is
+
+        :param flags: Chrome flags to be added to NSTBrowser query
+        :return: CDP URL
+        """
         cdp_url = self.cdp_url
         if self.nstbrowser_mode:
             if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
@@ -74,7 +99,12 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    def fetch(self, url) -> Response:
+    def fetch(self, url: str) -> Response:
+        """Opens up the browser and do your request based on your chosen options.
+
+        :param url: Target url.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         if not self.stealth:
             from playwright.sync_api import sync_playwright
         else:

From 282b130864d0a10ba77a6d094dc61d0650b5f2d5 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:25:34 +0200
Subject: [PATCH 041/118] Static Engine - Doc strings and type annotations
 stuff

---
 scrapling/engines/static.py | 66 ++++++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 12 deletions(-)

diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 35b575a..6c6bf84 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -8,19 +8,28 @@ from httpx._models import Response as httpxResponse
 
 
 class StaticEngine:
-    def __init__(
-            self,
-            follow_redirects: bool = True,
-            timeout: Optional[Union[int, float]] = None,
-            adaptor_arguments: Dict = None
-    ):
+    def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
+        """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
+
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
         self.timeout = timeout
         self.follow_redirects = bool(follow_redirects)
         self._extra_headers = generate_headers(browser_mode=False)
         self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
     @staticmethod
-    def _headers_job(headers, url, stealth):
+    def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:
+        """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
+        finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
+
+        :param headers: Current headers in the request if the user passed any
+        :param url: The Target URL.
+        :param stealth: Whether stealth mode is enabled or not.
+        :return: A dictionary of the new headers.
+        """
         headers = headers or {}
 
         # Validate headers
@@ -35,7 +44,12 @@ class StaticEngine:
 
         return headers
 
-    def _prepare_response(self, response: httpxResponse):
+    def _prepare_response(self, response: httpxResponse) -> Response:
+        """Takes httpx response and generates `Response` object from it.
+
+        :param response: httpx response object
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         return Response(
             url=str(response.url),
             text=response.text,
@@ -49,22 +63,50 @@ class StaticEngine:
             adaptor_arguments=self.adaptor_arguments
         )
 
-    def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+    def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP GET request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
         return self._prepare_response(request)
 
-    def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+    def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP POST request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
         return self._prepare_response(request)
 
-    def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+    def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP DELETE request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
         return self._prepare_response(request)
 
-    def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
+    def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP PUT request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
         headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
         request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
         return self._prepare_response(request)

From b8afb390aeeade25bb8b6ab9a3d9d086e921c2a4 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:43:16 +0200
Subject: [PATCH 042/118] Renaming `fetcher` file to `fetchers` for clarity

---
 scrapling/__init__.py                 | 2 +-
 scrapling/{fetcher.py => fetchers.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename scrapling/{fetcher.py => fetchers.py} (100%)

diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 8323d50..d33f63a 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -1,5 +1,5 @@
 # Declare top-level shortcuts
-from scrapling.fetcher import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
+from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
 from scrapling.parser import Adaptor, Adaptors
 from scrapling.core.custom_types import TextHandler, AttributesHandler
 
diff --git a/scrapling/fetcher.py b/scrapling/fetchers.py
similarity index 100%
rename from scrapling/fetcher.py
rename to scrapling/fetchers.py

From 2ff0fbb0e554eb52148d687934eba83e7c523727 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:46:40 +0200
Subject: [PATCH 043/118] Engines utils - Navigation functions

- Adding doc strings and type annotations stuff.
- Making error messages more detailed.
- Renaming the `construct_websocket_url` function for more clarity.
---
 scrapling/engines/pw.py                  |  4 +--
 scrapling/engines/toolbelt/__init__.py   |  2 +-
 scrapling/engines/toolbelt/navigation.py | 33 ++++++++++++++++++------
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index aa2e0c2..67ea49e 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -10,7 +10,7 @@ from scrapling.engines.toolbelt import (
     intercept_route,
     generate_headers,
     check_type_validity,
-    construct_websocket_url,
+    construct_cdp_url,
     generate_convincing_referer,
 )
 
@@ -95,7 +95,7 @@ class PlaywrightEngine:
                     'config': json.dumps(query),
                     # 'token': ''
                 }
-            cdp_url = construct_websocket_url(cdp_url, config)
+            cdp_url = construct_cdp_url(cdp_url, config)
 
         return cdp_url
 
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 375e304..7e8368a 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -15,5 +15,5 @@ from .custom import (
 from .navigation import (
     js_bypass_path,
     intercept_route,
-    construct_websocket_url,
+    construct_cdp_url,
 )
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 5a8192b..111b17c 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -8,27 +8,39 @@ from urllib.parse import urlparse, urlencode
 from playwright.sync_api import Route
 
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+from scrapling.core._types import Union, Dict
 
 
-def intercept_route(route: Route):
+def intercept_route(route: Route) -> Union[Route, None]:
+    """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+
+    :param route: PlayWright `Route` object of the current page
+    :return: PlayWright `Route` object
+    """
     if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
         logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
         return route.abort()
     return route.continue_()
 
 
-def construct_websocket_url(base_url, query_params):
-    # Validate the base URL structure
+def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:
+    """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
+
+    :param cdp_url: The target URL.
+    :param query_params: A dictionary of the parameters to add.
+    :return: The new CDP URL.
+    """
     try:
-        parsed = urlparse(base_url)
+        # Validate the base URL structure
+        parsed = urlparse(cdp_url)
 
         # Check scheme
         if parsed.scheme not in ('ws', 'wss'):
-            raise ValueError("URL must use 'ws://' or 'wss://' scheme")
+            raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
 
         # Validate hostname and port
         if not parsed.netloc:
-            raise ValueError("Invalid hostname")
+            raise ValueError("Invalid hostname for the CDP URL")
 
         # Ensure path starts with /
         path = parsed.path
@@ -46,9 +58,14 @@ def construct_websocket_url(base_url, query_params):
         return validated_base
 
     except Exception as e:
-        raise ValueError(f"Invalid WebSocket URL: {str(e)}")
+        raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-def js_bypass_path(filename):
+def js_bypass_path(filename: str) -> str:
+    """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
+
+    :param filename: The base filename of the JS file.
+    :return: The full path of the JS file.
+    """
     current_directory = os.path.dirname(__file__)
     return os.path.join(current_directory, 'bypasses', filename)

From e270b0081126eeb93b15a9c798cbcd28922791b2 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 23:17:20 +0200
Subject: [PATCH 044/118] Toolbelt subpackage custom functions general
 adjustments

---
 scrapling/engines/toolbelt/custom.py     | 16 ++++++++++++----
 scrapling/engines/toolbelt/navigation.py |  5 +++--
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index c06d68d..9dee07f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -5,13 +5,14 @@ import inspect
 import logging
 from dataclasses import dataclass, field
 
-from scrapling.parser import Adaptor, SQLiteStorageSystem
 from scrapling.core.utils import setup_basic_logging
-from scrapling.core._types import Any, List, Type, Union, Optional, Dict
+from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
 
 
 @dataclass(frozen=True)
 class Response:
+    """This class is returned by all engines as a way to unify response type between different libraries."""
     url: str
     text: str
     content: bytes
@@ -24,7 +25,8 @@ class Response:
     adaptor_arguments: Dict = field(default_factory=dict)
 
     @property
-    def adaptor(self):
+    def adaptor(self) -> Union[Adaptor, None]:
+        """Generate Adaptor instance from this response if possible, otherwise return None"""
         if self.content:
             return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
         elif self.text:
@@ -67,7 +69,13 @@ class BaseFetcher:
         setup_basic_logging(level='debug' if debug else 'info')
 
 
-def check_if_engine_usable(engine):
+def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
+    """This function check if the passed engine can be used by a Fetcher-type class or not.
+
+    :param engine: The engine class itself
+    :return: The engine class again if all checks out, otherwise raises error
+    :raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
+    """
     # if isinstance(engine, type):
     #     raise TypeError("Expected an engine instance, not a class definition of the engine")
 
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 111b17c..2af7b86 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -5,10 +5,11 @@ Functions related to files and URLs
 import os
 import logging
 from urllib.parse import urlparse, urlencode
-from playwright.sync_api import Route
 
-from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 from scrapling.core._types import Union, Dict
+from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+
+from playwright.sync_api import Route
 
 
 def intercept_route(route: Route) -> Union[Route, None]:

From 544d23b696f4864e3532bfbbc0482dc3b6b6c1c1 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 3 Nov 2024 23:31:09 +0200
Subject: [PATCH 045/118] Toolbelt fingerprint functions - Doc strings and type
 annotations stuff

---
 scrapling/engines/toolbelt/__init__.py     |  1 -
 scrapling/engines/toolbelt/fingerprints.py | 37 +++++++++++++++-------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 7e8368a..08b559b 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -2,7 +2,6 @@ from .fingerprints import (
     get_os_name,
     generate_headers,
     generate_convincing_referer,
-    generate_suitable_fingerprint,
 )
 from .custom import (
     Response,
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
index 3969b0a..76f40da 100644
--- a/scrapling/engines/toolbelt/fingerprints.py
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -4,27 +4,32 @@ Functions related to generating headers and fingerprints generally
 
 import platform
 
+from scrapling.core._types import Union, Dict
+
 from tldextract import extract
-from browserforge.fingerprints import FingerprintGenerator
 from browserforge.headers import HeaderGenerator, Browser
+from browserforge.fingerprints import FingerprintGenerator, Fingerprint
 
 
-def generate_convincing_referer(url):
-    """
-    Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
+def generate_convincing_referer(url: str) -> str:
+    """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
     >>> generate_convincing_referer('https://www.somewebsite.com/blah')
     'https://www.google.com/search?q=somewebsite'
 
     :param url: The URL you are about to fetch.
-    :return:
+    :return: Google's search URL of the domain name
     """
     website_name = extract(url).domain
     return f'https://www.google.com/search?q={website_name}'
 
 
-def get_os_name():
-    # Get the OS name in the same format needed for browserforge
+def get_os_name() -> Union[str, None]:
+    """Get the current OS name in the same format needed for browserforge
+
+    :return: Current OS name or `None` otherwise
+    """
+    #
     os_name = platform.system()
     return {
         'Linux': 'linux',
@@ -35,17 +40,25 @@ def get_os_name():
     }.get(os_name)
 
 
-def generate_suitable_fingerprint():
-    # This would be for Browserforge playwright injector
-    os_name = get_os_name()
+def generate_suitable_fingerprint() -> Fingerprint:
+    """Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.
+
+    This function was originally created to test Browserforge's injector.
+    :return: `Fingerprint` object
+    """
     return FingerprintGenerator(
         browser=[Browser(name='chrome', min_version=128)],
-        os=os_name,  # None is ignored
+        os=get_os_name(),  # None is ignored
         device='desktop'
     ).generate()
 
 
-def generate_headers(browser_mode=False):
+def generate_headers(browser_mode: bool = False) -> Dict:
+    """Generate real browser-like headers using browserforge's generator
+
+    :param browser_mode: If enabled, the headers created are used for playwright so it have to match everything
+    :return: A dictionary of the generated headers
+    """
     if browser_mode:
         # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
         # So we don't raise any inconsistency red flags while websites fingerprinting us

From bd612bb0e26ee37f68d2569d887c4346c662338b Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 4 Nov 2024 20:16:04 +0200
Subject: [PATCH 046/118] Updating the `disable_resources` doc-string

---
 scrapling/engines/camo.py | 2 +-
 scrapling/engines/pw.py   | 2 +-
 scrapling/fetchers.py     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 1514424..8e944fd 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -25,7 +25,7 @@ class CamoufoxEngine:
         :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 67ea49e..ef493dd 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -36,7 +36,7 @@ class PlaywrightEngine:
         """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
 
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 80b913a..6f569ee 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -80,7 +80,7 @@ class StealthyFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
@@ -136,7 +136,7 @@ class PlayWrightFetcher(BaseFetcher):
         """Opens up a browser and do your request based on your chosen options below.
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.

From e594bba0c4e56a2a89806f31b80872ca7dc041a5 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 01:19:50 +0200
Subject: [PATCH 047/118] Testing some of the new tests on Github action

---
 .github/workflows/tests.yml                  | 11 ++++
 tests/requirements.txt                       |  6 ++-
 tests/test_fetchers/__init__.py              |  1 +
 tests/test_fetchers/test_stealthy_fetcher.py | 57 ++++++++++++++++++++
 tox.ini                                      |  8 ++-
 5 files changed, 80 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_fetchers/__init__.py
 create mode 100644 tests/test_fetchers/test_stealthy_fetcher.py

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 77dd783..d6d6ef6 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -44,6 +44,17 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Install Playwright Dependencies
+      run: |
+        pip install playwright
+        playwright install chromium
+        playwright install-deps chromium
+
+    - name: Install Camoufox Dependencies
+      run: |
+        pip install camoufox
+        python -m camoufox fetch --browserforge
+
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
diff --git a/tests/requirements.txt b/tests/requirements.txt
index cffeec6..fd7ef96 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,2 +1,6 @@
 pytest
-pytest-cov
\ No newline at end of file
+pytest-cov
+playwright
+camoufox
+pytest-httpbin
+pytest-playwright
diff --git a/tests/test_fetchers/__init__.py b/tests/test_fetchers/__init__.py
new file mode 100644
index 0000000..1014360
--- /dev/null
+++ b/tests/test_fetchers/__init__.py
@@ -0,0 +1 @@
+# Because I'm too lazy to mock requests :)
diff --git a/tests/test_fetchers/test_stealthy_fetcher.py b/tests/test_fetchers/test_stealthy_fetcher.py
new file mode 100644
index 0000000..b6c6182
--- /dev/null
+++ b/tests/test_fetchers/test_stealthy_fetcher.py
@@ -0,0 +1,57 @@
+import unittest
+import pytest_httpbin
+
+from scrapling import StealthyFetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+# @pytest_httpbin.use_class_based_httpbin_secure
+class TestParser(unittest.TestCase):
+    def setUp(self):
+        self.fetcher = StealthyFetcher(auto_match=False)
+        # httpsbin = self.httpbin_secure.url
+        url = self.httpbin.url
+        self.status_200 = f'{url}/status/200'
+        self.status_404 = f'{url}/status/404'
+        self.status_501 = f'{url}/status/501'
+        self.basic_url = f'{url}/get'
+        self.html_url = f'{url}/html'
+
+    def test_basic_fetch(self):
+        """Test doing basic fetch request with multiple statuses"""
+        self.assertEqual(self.fetcher.fetch(self.status_200).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.status_404).status, 404)
+        self.assertEqual(self.fetcher.fetch(self.status_501).status, 501)
+
+    def test_networkidle(self):
+        """Test if waiting for `networkidle` make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.basic_url, network_idle=True).status, 200)
+
+    def test_blocking_resources(self):
+        """Test if blocking resources make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.basic_url, block_images=True).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.basic_url, disable_resources=True).status, 200)
+
+    def test_waiting_selector(self):
+        """Test if waiting for a selector make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+
+    def test_automation(self):
+        """Test if automation break the code or not"""
+        def scroll_page(page):
+            page.mouse.wheel(10, 0)
+            page.mouse.move(100, 400)
+            page.mouse.up()
+            return page
+
+        self.assertEqual(self.fetcher.fetch(self.html_url, page_action=scroll_page).status, 200)
+
+    def test_properties(self):
+        """Test if different arguments breaks the code or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status, 200)
+
+    def test_infinite_timeout(self):
+        """Test if infinite timeout breaks the code or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)
diff --git a/tox.ini b/tox.ini
index 77d25dd..fdf42c3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,14 +4,18 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = pre-commit,py37,py38,py39,py310,py311,py312
+envlist = pre-commit,py{38,39,310,311,312,313}
 
 [testenv]
 usedevelop = True
 changedir = tests
 deps =
     -r{toxinidir}/tests/requirements.txt
-commands = pytest --cov=scrapling --cov-report=xml
+commands =
+    playwright install chromium
+    playwright install-deps chromium
+    camoufox fetch --browserforge
+    pytest --cov=scrapling --cov-report=xml
 
 [testenv:pre-commit]
 basepython = python3

From 3b25a248fcfcfce00f1c683dd45d4d2b5dcc39e1 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:04:49 +0200
Subject: [PATCH 048/118] Fixing tests action

---
 .github/workflows/tests.yml | 10 ++++------
 tests/requirements.txt      |  6 +++---
 tox.ini                     |  2 --
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d6d6ef6..dc5d690 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -7,7 +7,10 @@ concurrency:
 
 jobs:
   tests:
+    timeout-minutes: 60
     runs-on: ${{ matrix.os }}
+    container:
+      image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
     strategy:
       fail-fast: false
       matrix:
@@ -43,12 +46,7 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-
-    - name: Install Playwright Dependencies
-      run: |
-        pip install playwright
-        playwright install chromium
-        playwright install-deps chromium
+        cache: 'pip'
 
     - name: Install Camoufox Dependencies
       run: |
diff --git a/tests/requirements.txt b/tests/requirements.txt
index fd7ef96..46fd0ec 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,6 +1,6 @@
-pytest
+pytest>=2.8.0,<9
 pytest-cov
 playwright
 camoufox
-pytest-httpbin
-pytest-playwright
+pytest-httpbin==2.1.0
+httpbin~=0.10.0
diff --git a/tox.ini b/tox.ini
index fdf42c3..97b9508 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,8 +12,6 @@ changedir = tests
 deps =
     -r{toxinidir}/tests/requirements.txt
 commands =
-    playwright install chromium
-    playwright install-deps chromium
     camoufox fetch --browserforge
     pytest --cov=scrapling --cov-report=xml
 

From 7fd50a4b9459c87ba61422cf1212a6c9b669c707 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:06:45 +0200
Subject: [PATCH 049/118] Update tests.yml

---
 .github/workflows/tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index dc5d690..237feea 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,7 +46,6 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        cache: 'pip'
 
     - name: Install Camoufox Dependencies
       run: |

From a5ffc0be5d45e4d2b7f4cbdca309a10bf5498fbf Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:09:32 +0200
Subject: [PATCH 050/118] Update tests.yml

---
 .github/workflows/tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 237feea..f39f4fd 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -49,8 +49,9 @@ jobs:
 
     - name: Install Camoufox Dependencies
       run: |
-        pip install camoufox
-        python -m camoufox fetch --browserforge
+        python3 -m pip install --upgrade pip
+        python3 -m pip install camoufox
+        python3 -m camoufox fetch --browserforge
 
     - name: Run tests
       env: ${{ matrix.env }}

From 72b36d6a26ecc59c4b89e70dc9ca8675df64ccdd Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:22:21 +0200
Subject: [PATCH 051/118] Fix werkzeug issue and try caching

---
 .github/workflows/tests.yml | 19 +++++++++++++++++--
 tests/requirements.txt      |  1 +
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f39f4fd..5a11dc2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,6 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     container:
       image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
+      options: --user 1001
     strategy:
       fail-fast: false
       matrix:
@@ -46,13 +47,27 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.py
+          requirements*.txt
+          tox.ini
 
     - name: Install Camoufox Dependencies
       run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install camoufox
         python3 -m camoufox fetch --browserforge
 
+    # Cache tox environments
+    - name: Cache tox environments
+      uses: actions/cache@v3
+      with:
+        path: .tox
+        # Include python version and os in cache key
+        key: tox-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'setup.py', 'requirements*.txt') }}
+        restore-keys: |
+          tox-${{ runner.os }}-py${{ matrix.python-version }}-
+          tox-${{ runner.os }}-
+
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 46fd0ec..d5f716f 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,5 +2,6 @@ pytest>=2.8.0,<9
 pytest-cov
 playwright
 camoufox
+werkzeug<3.0.0
 pytest-httpbin==2.1.0
 httpbin~=0.10.0

From 4121fe73c9b228b5b1745b7b9e41552c40332c92 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:27:30 +0200
Subject: [PATCH 052/118] Update tests.yml

---
 .github/workflows/tests.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5a11dc2..2f2f0e4 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -43,6 +43,13 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+
+    # Install lsb-release before setup-python so the cache work
+    - name: Install lsb-release
+      run: |
+        apt-get update
+        apt-get install -y lsb-release
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:

From 80cc0b77f30ad4762275f89ee7af65195324109a Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:31:34 +0200
Subject: [PATCH 053/118] Update tests.yml

---
 .github/workflows/tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2f2f0e4..c20b451 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,7 +11,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     container:
       image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
-      options: --user 1001
     strategy:
       fail-fast: false
       matrix:

From 33f44587e6463c2d64b27de47db2937386b01957 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:33:21 +0200
Subject: [PATCH 054/118] Update tests.yml

---
 .github/workflows/tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c20b451..6fa1d09 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,8 +46,8 @@ jobs:
     # Install lsb-release before setup-python so the cache work
     - name: Install lsb-release
       run: |
-        apt-get update
-        apt-get install -y lsb-release
+        sudo apt-get update
+        sudo apt-get install -y lsb-release
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5

From 6f7d925a2ac62805573a79431e0a7cdb8f8e4fdd Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:34:43 +0200
Subject: [PATCH 055/118] Revert "Update tests.yml"

This reverts commit 33f44587e6463c2d64b27de47db2937386b01957.
---
 .github/workflows/tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6fa1d09..c20b451 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,8 +46,8 @@ jobs:
     # Install lsb-release before setup-python so the cache work
     - name: Install lsb-release
       run: |
-        sudo apt-get update
-        sudo apt-get install -y lsb-release
+        apt-get update
+        apt-get install -y lsb-release
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5

From 1dd04f8250984df3a86639f1a1e9240824502105 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 02:39:15 +0200
Subject: [PATCH 056/118] Update tests.yml

---
 .github/workflows/tests.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c20b451..6d6efa8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -53,14 +53,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        cache: 'pip'
-        cache-dependency-path: |
-          setup.py
-          requirements*.txt
-          tox.ini
 
     - name: Install Camoufox Dependencies
       run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install camoufox
         python3 -m camoufox fetch --browserforge
 
     # Cache tox environments

From f200e9b290e1566348a70c7caa306ad0cd5c766a Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:28:12 +0200
Subject: [PATCH 057/118] looks like playwright need to run as normal user

---
 .github/workflows/tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6d6efa8..69575d6 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,6 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     container:
       image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
+      options: --user 1001
     strategy:
       fail-fast: false
       matrix:
@@ -46,8 +47,8 @@ jobs:
     # Install lsb-release before setup-python so the cache work
     - name: Install lsb-release
       run: |
-        apt-get update
-        apt-get install -y lsb-release
+        sudo apt-get update
+        sudo apt-get install -y lsb-release
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5

From 8933a99a6ed65021eb32079e40bf08025332e8ab Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:29:28 +0200
Subject: [PATCH 058/118] Update tests.yml

---
 .github/workflows/tests.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 69575d6..b87e5e0 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -44,12 +44,6 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    # Install lsb-release before setup-python so the cache work
-    - name: Install lsb-release
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y lsb-release
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:

From a60662af4a2a20f29adf1912df1a7e12ddb92f74 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:38:35 +0200
Subject: [PATCH 059/118] Update camo.py

---
 scrapling/engines/camo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 8e944fd..c90995b 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -66,6 +66,7 @@ class CamoufoxEngine:
                 os=get_os_name(),
                 block_webrtc=self.block_webrtc,
                 allow_webgl=self.allow_webgl,
+                i_know_what_im_doing=True,  # To turn warnings off with user configurations
         ) as browser:
             page = browser.new_page()
             page.set_default_navigation_timeout(self.timeout)

From d4358d32a8ec49abfda3f1ce7318ff19a1e0a627 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:38:38 +0200
Subject: [PATCH 060/118] Update tests.yml

---
 .github/workflows/tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b87e5e0..4dd61c2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -53,6 +53,7 @@ jobs:
       run: |
         python3 -m pip install --upgrade pip
         python3 -m pip install camoufox
+        python3 -m playwright install-deps
         python3 -m camoufox fetch --browserforge
 
     # Cache tox environments

From 1bbb5dd75a6ee1f28f0e68c1c614fea539d98342 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:41:58 +0200
Subject: [PATCH 061/118] Update tests.yml

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4dd61c2..74f019c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -52,8 +52,8 @@ jobs:
     - name: Install Camoufox Dependencies
       run: |
         python3 -m pip install --upgrade pip
+        python3 -m playwright install-deps firefox
         python3 -m pip install camoufox
-        python3 -m playwright install-deps
         python3 -m camoufox fetch --browserforge
 
     # Cache tox environments

From 92d002032daad2edb8061a3831efc8c025d3d7ad Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:44:10 +0200
Subject: [PATCH 062/118] Update tests.yml

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 74f019c..053bf45 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -52,7 +52,7 @@ jobs:
     - name: Install Camoufox Dependencies
       run: |
         python3 -m pip install --upgrade pip
-        python3 -m playwright install-deps firefox
+        playwright install-deps firefox
         python3 -m pip install camoufox
         python3 -m camoufox fetch --browserforge
 

From 202021478412cfd378d84ad5bdd4bf7da945d0c9 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:45:59 +0200
Subject: [PATCH 063/118] please work

---
 .github/workflows/tests.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 053bf45..c974438 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -51,9 +51,8 @@ jobs:
 
     - name: Install Camoufox Dependencies
       run: |
-        python3 -m pip install --upgrade pip
-        playwright install-deps firefox
-        python3 -m pip install camoufox
+        python3 -m pip install --upgrade pip playwright camoufox
+        python3 -m playwright install-deps firefox
         python3 -m camoufox fetch --browserforge
 
     # Cache tox environments

From 6f91d06b460b5690dc1af709e9a7b76fd9f24d4c Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:57:57 +0200
Subject: [PATCH 064/118] Update tests.yml

---
 .github/workflows/tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c974438..4fbef64 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -51,7 +51,8 @@ jobs:
 
     - name: Install Camoufox Dependencies
       run: |
-        python3 -m pip install --upgrade pip playwright camoufox
+        python3 -m pip install --upgrade pip
+        python3 -m pip install playwright camoufox
         python3 -m playwright install-deps firefox
         python3 -m camoufox fetch --browserforge
 

From 69cfbe45be8aab69532a6653b0fbeb4ccdef5068 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:59:39 +0200
Subject: [PATCH 065/118] Update tests.yml

---
 .github/workflows/tests.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4fbef64..4a2227d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,7 +11,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     container:
       image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
-      options: --user 1001
     strategy:
       fail-fast: false
       matrix:
@@ -51,8 +50,8 @@ jobs:
 
     - name: Install Camoufox Dependencies
       run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install playwright camoufox
+        python3 -m pip install --upgrade pip --user
+        python3 -m pip install playwright camoufox --user
         python3 -m playwright install-deps firefox
         python3 -m camoufox fetch --browserforge
 

From 2a275c436c0bc953bbdeb06d9f5faf937bb1f083 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 13:02:11 +0200
Subject: [PATCH 066/118] Update tests.yml

---
 .github/workflows/tests.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4a2227d..4f421a8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,8 +9,6 @@ jobs:
   tests:
     timeout-minutes: 60
     runs-on: ${{ matrix.os }}
-    container:
-      image: mcr.microsoft.com/playwright/python:v1.48.0-jammy
     strategy:
       fail-fast: false
       matrix:
@@ -50,8 +48,8 @@ jobs:
 
     - name: Install Camoufox Dependencies
       run: |
-        python3 -m pip install --upgrade pip --user
-        python3 -m pip install playwright camoufox --user
+        python3 -m pip install --upgrade pip
+        python3 -m pip install playwright camoufox
         python3 -m playwright install-deps firefox
         python3 -m camoufox fetch --browserforge
 

From ca4b5f205c489a023d41c47c146f20d9803fe48a Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 13:10:51 +0200
Subject: [PATCH 067/118] Since tests can finally now run in Github actions
 without issues

Let's miss it up again lol
---
 .github/workflows/tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4f421a8..be98a97 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -45,6 +45,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.py
+          requirements*.txt
+          tox.ini
 
     - name: Install Camoufox Dependencies
       run: |

From d50497dff38f75bc692dd7b4b3e76c1e072e69b4 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 22:56:57 +0200
Subject: [PATCH 068/118] Update .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index c7b104c..1c6a0b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@ __pycache__/
 .bootstrap
 .appveyor.token
 *.bak
+*.db
 
 # installation package
 *.egg-info/

From 009c2914aa323091a58b9a0e1f8b8b18e2f60f25 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 22:59:47 +0200
Subject: [PATCH 069/118] Fixing `re_first` logic in `Adaptors` type

---
 scrapling/parser.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 2d0ebf1..3416ec6 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -867,17 +867,18 @@ class Adaptors(List[Adaptor]):
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True):
         """Call the ``.re_first()`` method for each element in this list and return
-        their results flattened as List of TextHandler.
+        the first result or the default value otherwise.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
 
         """
-        results = [
-            n.text.re_first(regex, default, replace_entities) for n in self
-        ]
-        return flatten(results)
+        for n in self:
+            result = n.re_first(regex, None, replace_entities)
+            if result:
+                return result
+        return default
 
     # def __getattr__(self, name):
     #     if name in dir(self.__class__):

From 3622040440980b192874c956bc911df1fb9a3430 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 23:41:51 +0200
Subject: [PATCH 070/118] Adding `css_first` and `xpath_first` for easier usage

---
 README.md           |  6 +++---
 scrapling/parser.py | 52 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 8ac4e06..84e6ada 100644
--- a/README.md
+++ b/README.md
@@ -59,7 +59,7 @@ quotes = page.css('.quote').css('.text::text')  # Chained selectors
 quotes = [element.text for element in page.css('.quote').css('.text')]  # Slower than bulk query above
 
 # Get the first quote element
-quote = page.css('.quote').first  # or [0] or .get()
+quote = page.css_first('.quote')  # or page.css('.quote').first or [0] or .get()
 
 # Working with elements
 quote.html_content  # Inner HTML
@@ -244,8 +244,8 @@ To increase the complexity a little bit, let's say we want to get all books' dat
 ```python
 >>> for product in page.find_by_text('Tipping the Velvet').parent.parent.find_similar():
         print({
-            "name": product.css('h3 a::text')[0],
-            "price": product.css('.price_color')[0].re_first(r'[\d\.]+'),
+            "name": product.css_first('h3 a::text'),
+            "price": product.css_first('.price_color').re_first(r'[\d\.]+'),
             "stock": product.css('.availability::text')[-1].clean()
         })
 {'name': 'A Light in the ...', 'price': '51.77', 'stock': 'In stock'}
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 3416ec6..4f994b4 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -394,6 +394,58 @@ class Adaptor(SelectorsGeneration):
                 return self.__convert_results(score_table[highest_probability])
         return []
 
+    def css_first(self, selector: str, identifier: str = '',
+                  auto_match: bool = False, auto_save: bool = False, percentage: int = 0
+                  ) -> Union['Adaptors[Adaptor]', List, None]:
+        """Search current tree with CSS3 selectors and return the first result if possible, otherwise return `None`
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+        :param selector: The CSS3 selector to be used.
+        :param auto_match: Enabled will make function try to relocate the element if it was 'saved' before
+        :param identifier: A string that will be used to save/retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        try:
+            return self.css(selector, identifier, auto_match, auto_save, percentage)[0]
+        except (IndexError, TypeError,):
+            return None
+
+    def xpath_first(self, selector: str, identifier: str = '',
+                    auto_match: bool = False, auto_save: bool = False, percentage: int = 0, **kwargs: Any
+                    ) -> Union['Adaptors[Adaptor]', List, None]:
+        """Search current tree with XPath selectors and return the first result if possible, otherwise return `None`
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+         Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**
+
+        :param selector: The XPath selector to be used.
+        :param auto_match: Enabled will make function try to relocate the element if it was 'saved' before
+        :param identifier: A string that will be used to save/retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        try:
+            return self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs)[0]
+        except (IndexError, TypeError,):
+            return None
+
     def css(self, selector: str, identifier: str = '',
             auto_match: bool = False, auto_save: bool = False, percentage: int = 0
             ) -> Union['Adaptors[Adaptor]', List]:

From 24806aecdc079ce5a6621dc4e6c7ed84b87d9bd4 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Tue, 5 Nov 2024 23:41:57 +0200
Subject: [PATCH 071/118] Update .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 1c6a0b4..7890c42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@ __pycache__/
 .appveyor.token
 *.bak
 *.db
+*.db-*
 
 # installation package
 *.egg-info/

From 6497b1b5598b6b096d53b339b522798b76a118ab Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 02:20:33 +0200
Subject: [PATCH 072/118] Type correction

---
 scrapling/engines/pw.py | 2 +-
 scrapling/fetchers.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index ef493dd..a62a691 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -18,7 +18,7 @@ from scrapling.engines.toolbelt import (
 class PlaywrightEngine:
     def __init__(
             self, headless: Union[bool, str] = True,
-            disable_resources: Optional[bool] = False,
+            disable_resources: bool = False,
             useragent: Optional[str] = None,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 6f569ee..f0fa432 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -125,7 +125,7 @@ class PlayWrightFetcher(BaseFetcher):
         > Note that these are the main options with PlayWright but it can be mixed together.
     """
     def fetch(
-            self, url: str, headless: Union[bool, str] = True, disable_resources: Optional[List] = None,
+            self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
             page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
             hide_canvas: bool = True, disable_webgl: bool = False,

From 6cf506cb22aa84bc8f80b3e21bf28482130df281 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:24:56 +0200
Subject: [PATCH 073/118] PlaywrightEngine - Validate CDP URLs in all cases

---
 scrapling/engines/pw.py                  | 3 +++
 scrapling/engines/toolbelt/navigation.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index a62a691..b08195c 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -96,6 +96,9 @@ class PlaywrightEngine:
                     # 'token': ''
                 }
             cdp_url = construct_cdp_url(cdp_url, config)
+        else:
+            # To validate it
+            cdp_url = construct_cdp_url(cdp_url)
 
         return cdp_url
 
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 2af7b86..e03cfd7 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -6,7 +6,7 @@ import os
 import logging
 from urllib.parse import urlparse, urlencode
 
-from scrapling.core._types import Union, Dict
+from scrapling.core._types import Union, Dict, Optional
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 
 from playwright.sync_api import Route
@@ -24,7 +24,7 @@ def intercept_route(route: Route) -> Union[Route, None]:
     return route.continue_()
 
 
-def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:
+def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
     """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
 
     :param cdp_url: The target URL.

From 10bcdfb5961f0f4a32afa2f4fdb01b73f8071855 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:19:26 +0200
Subject: [PATCH 074/118] Moving bypasses folder deeper

---
 scrapling/engines/{ => toolbelt}/bypasses/navigator_plugins.js    | 0
 .../engines/{ => toolbelt}/bypasses/notification_permission.js    | 0
 scrapling/engines/{ => toolbelt}/bypasses/pdf_viewer.js           | 0
 .../engines/{ => toolbelt}/bypasses/playwright_fingerprint.js     | 0
 scrapling/engines/{ => toolbelt}/bypasses/screen_props.js         | 0
 scrapling/engines/{ => toolbelt}/bypasses/webdriver_fully.js      | 0
 scrapling/engines/{ => toolbelt}/bypasses/window_chrome.js        | 0
 7 files changed, 0 insertions(+), 0 deletions(-)
 rename scrapling/engines/{ => toolbelt}/bypasses/navigator_plugins.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/notification_permission.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/pdf_viewer.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/playwright_fingerprint.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/screen_props.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/webdriver_fully.js (100%)
 rename scrapling/engines/{ => toolbelt}/bypasses/window_chrome.js (100%)

diff --git a/scrapling/engines/bypasses/navigator_plugins.js b/scrapling/engines/toolbelt/bypasses/navigator_plugins.js
similarity index 100%
rename from scrapling/engines/bypasses/navigator_plugins.js
rename to scrapling/engines/toolbelt/bypasses/navigator_plugins.js
diff --git a/scrapling/engines/bypasses/notification_permission.js b/scrapling/engines/toolbelt/bypasses/notification_permission.js
similarity index 100%
rename from scrapling/engines/bypasses/notification_permission.js
rename to scrapling/engines/toolbelt/bypasses/notification_permission.js
diff --git a/scrapling/engines/bypasses/pdf_viewer.js b/scrapling/engines/toolbelt/bypasses/pdf_viewer.js
similarity index 100%
rename from scrapling/engines/bypasses/pdf_viewer.js
rename to scrapling/engines/toolbelt/bypasses/pdf_viewer.js
diff --git a/scrapling/engines/bypasses/playwright_fingerprint.js b/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js
similarity index 100%
rename from scrapling/engines/bypasses/playwright_fingerprint.js
rename to scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js
diff --git a/scrapling/engines/bypasses/screen_props.js b/scrapling/engines/toolbelt/bypasses/screen_props.js
similarity index 100%
rename from scrapling/engines/bypasses/screen_props.js
rename to scrapling/engines/toolbelt/bypasses/screen_props.js
diff --git a/scrapling/engines/bypasses/webdriver_fully.js b/scrapling/engines/toolbelt/bypasses/webdriver_fully.js
similarity index 100%
rename from scrapling/engines/bypasses/webdriver_fully.js
rename to scrapling/engines/toolbelt/bypasses/webdriver_fully.js
diff --git a/scrapling/engines/bypasses/window_chrome.js b/scrapling/engines/toolbelt/bypasses/window_chrome.js
similarity index 100%
rename from scrapling/engines/bypasses/window_chrome.js
rename to scrapling/engines/toolbelt/bypasses/window_chrome.js

From 9a7f489eb79375993a0933308a5868b6e488b861 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:20:05 +0200
Subject: [PATCH 075/118] make tests verbose for easier debugging later

---
 pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 9ec48c5..df7eb7e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = -p no:warnings --doctest-modules --ignore=setup.py
\ No newline at end of file
+addopts = -p no:warnings --doctest-modules --ignore=setup.py --verbose
\ No newline at end of file

From 4400363146c4db0df6ffbf64947571e60633b7c4 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:21:06 +0200
Subject: [PATCH 076/118] Renaming fetchers tests folder for readability

---
 tests/{test_fetchers => fetchers}/__init__.py                  | 0
 .../test_stealthy_fetcher.py => fetchers/test_camoufox.py}     | 3 +--
 2 files changed, 1 insertion(+), 2 deletions(-)
 rename tests/{test_fetchers => fetchers}/__init__.py (100%)
 rename tests/{test_fetchers/test_stealthy_fetcher.py => fetchers/test_camoufox.py} (96%)

diff --git a/tests/test_fetchers/__init__.py b/tests/fetchers/__init__.py
similarity index 100%
rename from tests/test_fetchers/__init__.py
rename to tests/fetchers/__init__.py
diff --git a/tests/test_fetchers/test_stealthy_fetcher.py b/tests/fetchers/test_camoufox.py
similarity index 96%
rename from tests/test_fetchers/test_stealthy_fetcher.py
rename to tests/fetchers/test_camoufox.py
index b6c6182..0249651 100644
--- a/tests/test_fetchers/test_stealthy_fetcher.py
+++ b/tests/fetchers/test_camoufox.py
@@ -6,10 +6,9 @@ from scrapling import StealthyFetcher
 
 @pytest_httpbin.use_class_based_httpbin
 # @pytest_httpbin.use_class_based_httpbin_secure
-class TestParser(unittest.TestCase):
+class TestStealthyFetcher(unittest.TestCase):
     def setUp(self):
         self.fetcher = StealthyFetcher(auto_match=False)
-        # httpsbin = self.httpbin_secure.url
         url = self.httpbin.url
         self.status_200 = f'{url}/status/200'
         self.status_404 = f'{url}/status/404'

From 4222626ec13a026b89ec5795d8c070211e72e0ca Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:21:18 +0200
Subject: [PATCH 077/118] Adding tests for `Fetcher` class

---
 tests/fetchers/test_httpx.py | 67 ++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 tests/fetchers/test_httpx.py

diff --git a/tests/fetchers/test_httpx.py b/tests/fetchers/test_httpx.py
new file mode 100644
index 0000000..2fcd585
--- /dev/null
+++ b/tests/fetchers/test_httpx.py
@@ -0,0 +1,67 @@
+import unittest
+import pytest_httpbin
+
+from scrapling import Fetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestFetcher(unittest.TestCase):
+    def setUp(self):
+        self.fetcher = Fetcher(auto_match=False)
+        url = self.httpbin.url
+        self.status_200 = f'{url}/status/200'
+        self.status_404 = f'{url}/status/404'
+        self.status_501 = f'{url}/status/501'
+        self.basic_url = f'{url}/get'
+        self.post_url = f'{url}/post'
+        self.put_url = f'{url}/put'
+        self.delete_url = f'{url}/delete'
+        self.html_url = f'{url}/html'
+
+    def test_basic_get(self):
+        """Test doing basic get request with multiple statuses"""
+        self.assertEqual(self.fetcher.get(self.status_200).status, 200)
+        self.assertEqual(self.fetcher.get(self.status_404).status, 404)
+        self.assertEqual(self.fetcher.get(self.status_501).status, 501)
+
+    def test_get_properties(self):
+        """Test if different arguments with GET request breaks the code or not"""
+        self.assertEqual(self.fetcher.get(self.status_200, stealthy_headers=True).status, 200)
+        self.assertEqual(self.fetcher.get(self.status_200, follow_redirects=True).status, 200)
+        self.assertEqual(self.fetcher.get(self.status_200, timeout=None).status, 200)
+        self.assertEqual(
+            self.fetcher.get(self.status_200, stealthy_headers=True, follow_redirects=True, timeout=None).status,
+            200
+        )
+
+    def test_post_properties(self):
+        """Test if different arguments with POST request breaks the code or not"""
+        self.assertEqual(self.fetcher.post(self.post_url, data={'key': 'value'}).status, 200)
+        self.assertEqual(self.fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status, 200)
+        self.assertEqual(self.fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status, 200)
+        self.assertEqual(self.fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status, 200)
+        self.assertEqual(
+            self.fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True, follow_redirects=True, timeout=None).status,
+            200
+        )
+
+    def test_put_properties(self):
+        """Test if different arguments with PUT request breaks the code or not"""
+        self.assertEqual(self.fetcher.put(self.put_url, data={'key': 'value'}).status, 200)
+        self.assertEqual(self.fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status, 200)
+        self.assertEqual(self.fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status, 200)
+        self.assertEqual(self.fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status, 200)
+        self.assertEqual(
+            self.fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True, follow_redirects=True, timeout=None).status,
+            200
+        )
+
+    def test_delete_properties(self):
+        """Test if different arguments with DELETE request breaks the code or not"""
+        self.assertEqual(self.fetcher.delete(self.delete_url, stealthy_headers=True).status, 200)
+        self.assertEqual(self.fetcher.delete(self.delete_url, follow_redirects=True).status, 200)
+        self.assertEqual(self.fetcher.delete(self.delete_url, timeout=None).status, 200)
+        self.assertEqual(
+            self.fetcher.delete(self.delete_url, stealthy_headers=True, follow_redirects=True, timeout=None).status,
+            200
+        )

From e4eb4427098d63a1ffbd4f639b745fdb5bd38660 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:21:55 +0200
Subject: [PATCH 078/118] Adding tests for `PlayWrightFetcher` class

---
 tests/fetchers/test_playwright.py | 68 +++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 tests/fetchers/test_playwright.py

diff --git a/tests/fetchers/test_playwright.py b/tests/fetchers/test_playwright.py
new file mode 100644
index 0000000..f0ba143
--- /dev/null
+++ b/tests/fetchers/test_playwright.py
@@ -0,0 +1,68 @@
+import unittest
+import pytest_httpbin
+
+from scrapling import PlayWrightFetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+# @pytest_httpbin.use_class_based_httpbin_secure
+class TestPlayWrightFetcher(unittest.TestCase):
+    def setUp(self):
+        self.fetcher = PlayWrightFetcher(auto_match=False)
+        url = self.httpbin.url
+        self.status_200 = f'{url}/status/200'
+        self.status_404 = f'{url}/status/404'
+        self.status_501 = f'{url}/status/501'
+        self.basic_url = f'{url}/get'
+        self.html_url = f'{url}/html'
+
+    def test_basic_fetch(self):
+        """Test doing basic fetch request with multiple statuses"""
+        self.assertEqual(self.fetcher.fetch(self.status_200).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.status_404).status, 404)
+        self.assertEqual(self.fetcher.fetch(self.status_501).status, 501)
+
+    def test_networkidle(self):
+        """Test if waiting for `networkidle` make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.basic_url, network_idle=True).status, 200)
+
+    def test_blocking_resources(self):
+        """Test if blocking resources make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.basic_url, disable_resources=True).status, 200)
+
+    def test_waiting_selector(self):
+        """Test if waiting for a selector make page does not finish loading or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+
+    def test_automation(self):
+        """Test if automation break the code or not"""
+        def scroll_page(page):
+            page.mouse.wheel(10, 0)
+            page.mouse.move(100, 400)
+            page.mouse.up()
+            return page
+
+        self.assertEqual(self.fetcher.fetch(self.html_url, page_action=scroll_page).status, 200)
+
+    def test_properties(self):
+        """Test if different arguments breaks the code or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, disable_webgl=True, hide_canvas=False).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, disable_webgl=False, hide_canvas=True).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, stealth=True).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0').status, 200)
+
+    def test_cdp_url(self):
+        """Test if it's going to try to connect to cdp url or not"""
+        with self.assertRaises(ValueError):
+            _ = self.fetcher.fetch(self.html_url, cdp_url='blahblah')
+
+        with self.assertRaises(ValueError):
+            _ = self.fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True)
+
+        with self.assertRaises(Exception):
+            # There's no type for this error in PlayWright, it's just `Error`
+            _ = self.fetcher.fetch(self.html_url, cdp_url='ws://blahblah')
+
+    def test_infinite_timeout(self):
+        """Test if infinite timeout breaks the code or not"""
+        self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)

From 5548941cf1d154051d3edb2bac13129c80e7763f Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:28:43 +0200
Subject: [PATCH 079/118] Correcting return type for `css_first` and
 `xpath_first`

---
 scrapling/parser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 4f994b4..52d9bcf 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -396,7 +396,7 @@ class Adaptor(SelectorsGeneration):
 
     def css_first(self, selector: str, identifier: str = '',
                   auto_match: bool = False, auto_save: bool = False, percentage: int = 0
-                  ) -> Union['Adaptors[Adaptor]', List, None]:
+                  ) -> Union['Adaptor', 'TextHandler', None]:
         """Search current tree with CSS3 selectors and return the first result if possible, otherwise return `None`
 
         **Important:
@@ -421,7 +421,7 @@ class Adaptor(SelectorsGeneration):
 
     def xpath_first(self, selector: str, identifier: str = '',
                     auto_match: bool = False, auto_save: bool = False, percentage: int = 0, **kwargs: Any
-                    ) -> Union['Adaptors[Adaptor]', List, None]:
+                    ) -> Union['Adaptor', 'TextHandler', None]:
         """Search current tree with XPath selectors and return the first result if possible, otherwise return `None`
 
         **Important:

From d632bb14af40abdb576a834c6704b19d1f58d815 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:30:41 +0200
Subject: [PATCH 080/118] Moving parser tests to a better structure

and covering a bit more parts
---
 tests/parser/__init__.py                      |  0
 tests/parser/test_automatch.py                | 56 +++++++++++++++++++
 .../test_general.py}                          | 52 +----------------
 3 files changed, 57 insertions(+), 51 deletions(-)
 create mode 100644 tests/parser/__init__.py
 create mode 100644 tests/parser/test_automatch.py
 rename tests/{test_parser_functions.py => parser/test_general.py} (82%)

diff --git a/tests/parser/__init__.py b/tests/parser/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/parser/test_automatch.py b/tests/parser/test_automatch.py
new file mode 100644
index 0000000..1e78e87
--- /dev/null
+++ b/tests/parser/test_automatch.py
@@ -0,0 +1,56 @@
+import unittest
+
+from scrapling import Adaptor
+
+
+class TestParserAutoMatch(unittest.TestCase):
+
+    def test_element_relocation(self):
+        """Test relocating element after structure change"""
+        original_html = '''
+                <div class="container">
+                    <section class="products">
+                        <article class="product" id="p1">
+                            <h3>Product 1</h3>
+                            <p class="description">Description 1</p>
+                        </article>
+                        <article class="product" id="p2">
+                            <h3>Product 2</h3>
+                            <p class="description">Description 2</p>
+                        </article>
+                    </section>
+                </div>
+                '''
+        changed_html = '''
+                <div class="new-container">
+                    <div class="product-wrapper">
+                        <section class="products">
+                            <article class="product new-class" data-id="p1">
+                                <div class="product-info">
+                                    <h3>Product 1</h3>
+                                    <p class="new-description">Description 1</p>
+                                </div>
+                            </article>
+                            <article class="product new-class" data-id="p2">
+                                <div class="product-info">
+                                    <h3>Product 2</h3>
+                                    <p class="new-description">Description 2</p>
+                                </div>
+                            </article>
+                        </section>
+                    </div>
+                </div>
+                '''
+
+        old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
+        new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
+
+        # 'p1' was used as ID and now it's not and all the path elements have changes
+        # Also at the same time testing auto-match vs combined selectors
+        _ = old_page.css('#p1, #p2', auto_save=True)[0]
+        relocated = new_page.css('#p1', auto_match=True)
+
+        self.assertIsNotNone(relocated)
+        self.assertEqual(relocated[0].attrib['data-id'], 'p1')
+        self.assertTrue(relocated[0].has_class('new-class'))
+        self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
diff --git a/tests/test_parser_functions.py b/tests/parser/test_general.py
similarity index 82%
rename from tests/test_parser_functions.py
rename to tests/parser/test_general.py
index 88013ad..4e746ec 100644
--- a/tests/test_parser_functions.py
+++ b/tests/parser/test_general.py
@@ -112,7 +112,7 @@ class TestParser(unittest.TestCase):
 
     def test_find_similar_elements(self):
         """Test Finding similar elements of an element"""
-        first_product = self.page.css('.product')[0]
+        first_product = self.page.css_first('.product')
         similar_products = first_product.find_similar()
         self.assertEqual(len(similar_products), 2)
 
@@ -265,56 +265,6 @@ class TestParser(unittest.TestCase):
         self.assertEqual(attr_json, {'jsonable': 'data'})
         self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
 
-    def test_element_relocation(self):
-        """Test relocating element after structure change"""
-        original_html = '''
-                <div class="container">
-                    <section class="products">
-                        <article class="product" id="p1">
-                            <h3>Product 1</h3>
-                            <p class="description">Description 1</p>
-                        </article>
-                        <article class="product" id="p2">
-                            <h3>Product 2</h3>
-                            <p class="description">Description 2</p>
-                        </article>
-                    </section>
-                </div>
-                '''
-        changed_html = '''
-                <div class="new-container">
-                    <div class="product-wrapper">
-                        <section class="products">
-                            <article class="product new-class" data-id="p1">
-                                <div class="product-info">
-                                    <h3>Product 1</h3>
-                                    <p class="new-description">Description 1</p>
-                                </div>
-                            </article>
-                            <article class="product new-class" data-id="p2">
-                                <div class="product-info">
-                                    <h3>Product 2</h3>
-                                    <p class="new-description">Description 2</p>
-                                </div>
-                            </article>
-                        </section>
-                    </div>
-                </div>
-                '''
-
-        old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
-        new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
-
-        # 'p1' was used as ID and now it's not and all the path elements have changes
-        # Also at the same time testing auto-match vs combined selectors
-        _ = old_page.css('#p1, #p2', auto_save=True)[0]
-        relocated = new_page.css('#p1', auto_match=True)
-
-        self.assertIsNotNone(relocated)
-        self.assertEqual(relocated[0].attrib['data-id'], 'p1')
-        self.assertTrue(relocated[0].has_class('new-class'))
-        self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
-
     def test_performance(self):
         """Test parsing and selecting speed"""
         import time

From cd1ff846a7fff2546c01d4f259d8854dc66e07a3 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:31:57 +0200
Subject: [PATCH 081/118] Update ROADMAP.md

---
 ROADMAP.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ROADMAP.md b/ROADMAP.md
index 5847c87..ed617b7 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -1,6 +1,6 @@
 ## TODOs
-- [ ] Add more tests and increase the code coverage.
-- [ ] Structure the tests folder in a better way.
+- [x] Add more tests and increase the code coverage.
+- [x] Structure the tests folder in a better way.
 - [ ] Add more documentation.
 - [x] Add the browsing ability.
 - [ ] Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.

From fc30cfed4ce67c22b316bb076543cd7e0a789b7c Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:36:46 +0200
Subject: [PATCH 082/118] Update tests.yml

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index be98a97..c77d16c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -55,7 +55,7 @@ jobs:
       run: |
         python3 -m pip install --upgrade pip
         python3 -m pip install playwright camoufox
-        python3 -m playwright install-deps firefox
+        python3 -m playwright install-deps chromium firefox
         python3 -m camoufox fetch --browserforge
 
     # Cache tox environments

From c30853ee2c1834de083e33c7d6c1dcb273c3a6b5 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:39:38 +0200
Subject: [PATCH 083/118] Update tox.ini

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 97b9508..9535007 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,6 +12,7 @@ changedir = tests
 deps =
     -r{toxinidir}/tests/requirements.txt
 commands =
+    playwright install-deps chromium firefox
     camoufox fetch --browserforge
     pytest --cov=scrapling --cov-report=xml
 

From b542b5cc39ce4e24b5db64160f7753d48f982da2 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:50:14 +0200
Subject: [PATCH 084/118] Recaching TOX environments

---
 .github/workflows/tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c77d16c..5628c99 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -64,10 +64,10 @@ jobs:
       with:
         path: .tox
         # Include python version and os in cache key
-        key: tox-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'setup.py', 'requirements*.txt') }}
+        key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'setup.py', 'requirements*.txt') }}
         restore-keys: |
-          tox-${{ runner.os }}-py${{ matrix.python-version }}-
-          tox-${{ runner.os }}-
+          tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-
+          tox-v1-${{ runner.os }}-
 
     - name: Run tests
       env: ${{ matrix.env }}

From a1513ba7690075760f2e2ebf3813113485d6780e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:54:52 +0200
Subject: [PATCH 085/118] Update tests.yml

---
 .github/workflows/tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5628c99..1e61a4f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -55,6 +55,7 @@ jobs:
       run: |
         python3 -m pip install --upgrade pip
         python3 -m pip install playwright camoufox
+        python3 -m playwright install chromium
         python3 -m playwright install-deps chromium firefox
         python3 -m camoufox fetch --browserforge
 

From 5ad21260305cafa1989d8737f3f15f57368046cf Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 19:56:39 +0200
Subject: [PATCH 086/118] Small speed boost for fetching by some caching

---
 scrapling/engines/toolbelt/fingerprints.py | 3 +++
 scrapling/engines/toolbelt/navigation.py   | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
index 76f40da..71b8e84 100644
--- a/scrapling/engines/toolbelt/fingerprints.py
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -4,6 +4,7 @@ Functions related to generating headers and fingerprints generally
 
 import platform
 
+from scrapling.core.utils import cache
 from scrapling.core._types import Union, Dict
 
 from tldextract import extract
@@ -11,6 +12,7 @@ from browserforge.headers import HeaderGenerator, Browser
 from browserforge.fingerprints import FingerprintGenerator, Fingerprint
 
 
+@cache(None, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -24,6 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
+@cache(None, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index e03cfd7..cf73a39 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -6,6 +6,7 @@ import os
 import logging
 from urllib.parse import urlparse, urlencode
 
+from scrapling.core.utils import cache
 from scrapling.core._types import Union, Dict, Optional
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 
@@ -62,6 +63,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
+@cache(None, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

From a0a3e62075b1a349525cdf350a5f6de379feb85d Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 20:14:42 +0200
Subject: [PATCH 087/118] Adding `addons` and `humanize` arguments to
 StealthyFetcher

---
 scrapling/engines/camo.py | 12 +++++++++---
 scrapling/fetchers.py     |  8 ++++++--
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index c90995b..fd43e02 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -1,5 +1,5 @@
 import logging
-from scrapling.core._types import Union, Callable, Optional, Dict
+from scrapling.core._types import Union, Callable, Optional, Dict, List
 
 from scrapling.engines.toolbelt import (
     Response,
@@ -16,8 +16,8 @@ from camoufox.sync_api import Camoufox
 class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False,
-            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
+            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
             wait_selector_state: str = 'attached', adaptor_arguments: Dict = None
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -29,6 +29,8 @@ class CamoufoxEngine:
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
         :param network_idle: Wait for the page to not do do any requests.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
@@ -43,6 +45,8 @@ class CamoufoxEngine:
         self.block_webrtc = bool(block_webrtc)
         self.allow_webgl = bool(allow_webgl)
         self.network_idle = bool(network_idle)
+        self.addons = addons or []
+        self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
         if callable(page_action):
             self.page_action = page_action
@@ -66,6 +70,8 @@ class CamoufoxEngine:
                 os=get_os_name(),
                 block_webrtc=self.block_webrtc,
                 allow_webgl=self.allow_webgl,
+                addons=self.addons,
+                humanize=self.humanize,
                 i_know_what_im_doing=True,  # To turn warnings off with user configurations
         ) as browser:
             page = browser.new_page()
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index f0fa432..58ba896 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -70,8 +70,8 @@ class StealthyFetcher(BaseFetcher):
     """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False,
-            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
+            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
             wait_selector_state: str = 'attached',
     ) -> Response:
         """
@@ -84,6 +84,8 @@ class StealthyFetcher(BaseFetcher):
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
         :param network_idle: Wait for the page to not do do any requests.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
@@ -98,6 +100,8 @@ class StealthyFetcher(BaseFetcher):
             page_action=page_action,
             block_images=block_images,
             block_webrtc=block_webrtc,
+            addons=addons,
+            humanize=humanize,
             allow_webgl=allow_webgl,
             disable_resources=disable_resources,
             network_idle=network_idle,

From a1fd5ddc7e0d77f1a487df9d87f36a626b0c1777 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 20:18:44 +0200
Subject: [PATCH 088/118] Correcting `headless` argument type

---
 scrapling/core/_types.py | 2 +-
 scrapling/fetchers.py    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index f832320..f46dad4 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -3,7 +3,7 @@ Type definitions for type checking purposes.
 """
 
 from typing import (
-    Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING
+    Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING, Literal
 )
 
 try:
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 58ba896..835a8e4 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -1,4 +1,4 @@
-from scrapling.core._types import Dict, Optional, Union, Callable, List
+from scrapling.core._types import Dict, Optional, Union, Callable, List, Literal
 
 from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing
 from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable
@@ -69,7 +69,7 @@ class StealthyFetcher(BaseFetcher):
      Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
     """
     def fetch(
-            self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
+            self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
             wait_selector_state: str = 'attached',
@@ -77,7 +77,7 @@ class StealthyFetcher(BaseFetcher):
         """
         Opens up a browser and do your request based on your chosen options below.
         :param url: Target url.
-        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.

From 7af75f78a5651951f5f3554926b564fc997858cd Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 20:24:30 +0200
Subject: [PATCH 089/118] Better logic for `Adaptors` `re_first` method

---
 scrapling/parser.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 52d9bcf..b58e319 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -927,8 +927,7 @@ class Adaptors(List[Adaptor]):
 
         """
         for n in self:
-            result = n.re_first(regex, None, replace_entities)
-            if result:
+            for result in n.re(regex, replace_entities):
                 return result
         return default
 

From 7a84511b7a5f65bbf2f5b940ea05f5de977936db Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 20:47:21 +0200
Subject: [PATCH 090/118] Making `Adaptor` and `Adaptor` re/re_first arguments
 consistent with the TextHandler ones

---
 scrapling/parser.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index b58e319..f3d95b3 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -658,23 +658,28 @@ class Adaptor(SelectorsGeneration):
         else:
             return self.get_all_text(strip=True).json()
 
-    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True) -> 'List[str]':
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True,
+           clean_match: bool = False, case_sensitive: bool = False) -> 'List[str]':
         """Apply the given regex to the current text and return a list of strings with the matches.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
         """
-        return self.text.re(regex, replace_entities)
+        return self.text.re(regex, replace_entities, clean_match, case_sensitive)
 
-    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True):
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
-
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
         """
-        return self.text.re_first(regex, default, replace_entities)
+        return self.text.re_first(regex, default, replace_entities, clean_match, case_sensitive)
 
     def find_similar(
             self,
@@ -905,29 +910,34 @@ class Adaptors(List[Adaptor]):
         ]
         return self.__class__(flatten(results))
 
-    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True) -> 'List[str]':
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True,
+           clean_match: bool = False, case_sensitive: bool = False) -> 'List[str]':
         """Call the ``.re()`` method for each element in this list and return
         their results flattened as List of TextHandler.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
         """
         results = [
-            n.text.re(regex, replace_entities) for n in self
+            n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self
         ]
         return flatten(results)
 
-    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True):
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
         """Call the ``.re_first()`` method for each element in this list and return
         the first result or the default value otherwise.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
-
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
         """
         for n in self:
-            for result in n.re(regex, replace_entities):
+            for result in n.re(regex, replace_entities, clean_match, case_sensitive):
                 return result
         return default
 

From 5e848ef0460e7a72d97782066763c39705827b39 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 21:03:19 +0200
Subject: [PATCH 091/118] Adding new class type `TextHandlers`

---
 scrapling/core/custom_types.py | 49 ++++++++++++++++++++++++++++++++--
 scrapling/parser.py            |  4 ++-
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index bffc36f..f157879 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -3,7 +3,7 @@ from types import MappingProxyType
 from collections.abc import Mapping
 
 from scrapling.core.utils import _is_iterable, flatten
-from scrapling.core._types import Dict, List, Union, Pattern
+from scrapling.core._types import Dict, List, Union, Pattern, SupportsIndex
 
 from orjson import loads, dumps
 from w3lib.html import replace_entities as _replace_entities
@@ -69,7 +69,7 @@ class TextHandler(str):
         return [TextHandler(_replace_entities(s)) for s in results]
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False,):
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
@@ -83,6 +83,51 @@ class TextHandler(str):
         return result[0] if result else default
 
 
+class TextHandlers(List[TextHandler]):
+    """
+    The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
+    """
+    __slots__ = ()
+
+    def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers[TextHandler]"]:
+        lst = super().__getitem__(pos)
+        if isinstance(pos, slice):
+            return self.__class__(lst)
+        else:
+            return lst
+
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
+            case_sensitive: bool = False) -> 'List[str]':
+        """Call the ``.re()`` method for each element in this list and return
+        their results flattened as TextHandlers.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        """
+        results = [
+            n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
+        ]
+        return flatten(results)
+
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
+        """Call the ``.re_first()`` method for each element in this list and return
+        the first result or the default value otherwise.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param default: The default value to be returned if there is no match
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        """
+        for n in self:
+            for result in n.re(regex, replace_entities, clean_match, case_sensitive):
+                return result
+        return default
+
+
 class AttributesHandler(Mapping):
     """A read-only mapping to use instead of the standard dictionary for the speed boost but
      at the same time I use it to add more functionalities.
diff --git a/scrapling/parser.py b/scrapling/parser.py
index f3d95b3..583b1c5 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -3,7 +3,7 @@ from difflib import SequenceMatcher
 
 from scrapling.core.translator import HTMLTranslator
 from scrapling.core.mixins import SelectorsGeneration
-from scrapling.core.custom_types import TextHandler, AttributesHandler
+from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
 from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
 from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
 from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
@@ -158,6 +158,8 @@ class Adaptor(SelectorsGeneration):
             results = [self.__get_correct_result(n) for n in result]
             if all(isinstance(res, self.__class__) for res in results):
                 return Adaptors(results)
+            elif all(isinstance(res, TextHandler) for res in results):
+                return TextHandlers(results)
             return results
 
         return self.__get_correct_result(result)

From 6b40af426fc90361c7d868b3f68a8206f8b5a538 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Wed, 6 Nov 2024 22:14:33 +0200
Subject: [PATCH 092/118] Better logic for `css_first` and `xpath_first`

---
 scrapling/parser.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 583b1c5..fdf3bda 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -416,10 +416,9 @@ class Adaptor(SelectorsGeneration):
 
         :return: List as :class:`Adaptors`
         """
-        try:
-            return self.css(selector, identifier, auto_match, auto_save, percentage)[0]
-        except (IndexError, TypeError,):
-            return None
+        for element in self.css(selector, identifier, auto_match, auto_save, percentage):
+            return element
+        return None
 
     def xpath_first(self, selector: str, identifier: str = '',
                     auto_match: bool = False, auto_save: bool = False, percentage: int = 0, **kwargs: Any
@@ -443,10 +442,9 @@ class Adaptor(SelectorsGeneration):
 
         :return: List as :class:`Adaptors`
         """
-        try:
-            return self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs)[0]
-        except (IndexError, TypeError,):
-            return None
+        for element in self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs):
+            return element
+        return None
 
     def css(self, selector: str, identifier: str = '',
             auto_match: bool = False, auto_save: bool = False, percentage: int = 0

From b25d2ee79b4970d08688a82702793b7b2792541e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 7 Nov 2024 00:01:14 +0200
Subject: [PATCH 093/118] Adding new `find_all` and `find` functions to parser

---
 scrapling/parser.py | 70 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index fdf3bda..9eecc62 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -5,8 +5,8 @@ from scrapling.core.translator import HTMLTranslator
 from scrapling.core.mixins import SelectorsGeneration
 from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
 from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
-from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
-from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, _is_iterable, html_forbidden
+from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable
 
 from lxml import etree, html
 from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
@@ -542,6 +542,72 @@ class Adaptor(SelectorsGeneration):
         except (SelectorError, SelectorSyntaxError, etree.XPathError, etree.XPathEvalError):
             raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")
 
+    def find_all(self, *args, **kwargs) -> Union['Adaptors[Adaptor]', List]:
+        """Find elements by their tag name and filter them based on attributes for ease..
+
+        :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.
+        :param kwargs: The attributes you want to filter elements based on it.
+        :return: The `Adaptors` object of the elements or empty list
+        """
+        # Attributes that are Python reserved words and can't be used directly
+        # Ex: find_all('a', class="blah") -> find_all('a', class_="blah")
+        whitelisted = {
+            'id_': 'id',
+            'class_': 'class',
+        }
+
+        if not args and not kwargs:
+            raise TypeError('You have to pass something to search with, like tag name(s), tag attributes, or both.')
+
+        tags = set()
+        selectors = []
+        attributes = dict()
+        # Brace yourself for a wonderful journey!
+        for arg in args:
+            if type(arg) is str:
+                tags.add(arg)
+
+            elif type(arg) in [list, tuple, set]:
+                if not all(map(lambda x: type(x) is str, arg)):
+                    raise TypeError('Nested Iterables are not accepted, only iterables of tag names are accepted')
+                tags.update(set(arg))
+
+            elif type(arg) is dict:
+                if not all([(type(k) is str and type(v) is str) for k, v in arg.items()]):
+                    raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted')
+                attributes.update(arg)
+
+            else:
+                raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.')
+
+        if not all([(type(k) is str and type(v) is str) for k, v in kwargs.items()]):
+            raise TypeError('Only string values are accepted for arguments')
+        attributes.update(kwargs)
+
+        # It's easier and faster to build a selector than traversing the tree
+        tags = tags or ['']
+        for tag in tags:
+            selector = tag
+            for key, value in attributes.items():
+                key = whitelisted.get(key, key)
+                value = value.replace('"', r'\"')  # Escape double quotes in user input
+                # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
+                selector += '[{}="{}"]'.format(key, value)
+            selectors.append(selector)
+
+        return self.css(', '.join(selectors))
+
+    def find(self, *args, **kwargs) -> Union['Adaptor', None]:
+        """Find elements by their tag name and filter them based on attributes for ease then return the first result. Otherwise return `None`.
+
+        :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.
+        :param kwargs: The attributes you want to filter elements based on it.
+        :return: The `Adaptor` object of the element or `None` if the result didn't match
+        """
+        for element in self.find_all(*args, **kwargs):
+            return element
+        return None
+
     def __calculate_similarity_score(self, original: Dict, candidate: html.HtmlElement) -> float:
         """Used internally to calculate a score that shows how candidate element similar to the original one
 

From 846a185b4969fedc4b4fc7220ac493a8440c0830 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 7 Nov 2024 00:02:36 +0200
Subject: [PATCH 094/118] Updating parser tests to use `find` and `css_first`
 functions

---
 tests/parser/test_general.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py
index 4e746ec..6edfced 100644
--- a/tests/parser/test_general.py
+++ b/tests/parser/test_general.py
@@ -116,7 +116,7 @@ class TestParser(unittest.TestCase):
         similar_products = first_product.find_similar()
         self.assertEqual(len(similar_products), 2)
 
-        first_review = self.page.css('.review')[0]
+        first_review = self.page.find('div', class_='review')
         similar_high_rated_reviews = [
             review
             for review in first_review.find_similar()
@@ -197,7 +197,7 @@ class TestParser(unittest.TestCase):
         parent_siblings = parent.siblings
         self.assertEqual(len(parent_siblings), 1)
 
-        child = table.css('[data-id="1"]')[0]
+        child = table.find({'data-id': "1"})
         next_element = child.next
         self.assertEqual(next_element.attrib['data-id'], '2')
 
@@ -261,7 +261,7 @@ class TestParser(unittest.TestCase):
         key_value = list(products[0].attrib.search_values('1', partial=True))
         self.assertEqual(list(key_value[0].keys()), ['data-id'])
 
-        attr_json = self.page.css('#products')[0].attrib['schema'].json()
+        attr_json = self.page.css_first('#products').attrib['schema'].json()
         self.assertEqual(attr_json, {'jsonable': 'data'})
         self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
 

From 91f6b46e6dd7500c41201d35cb5462d4fdfae6f6 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 7 Nov 2024 00:31:14 +0200
Subject: [PATCH 095/118] Adding new `filter` and `search` functions to parser

---
 scrapling/parser.py | 43 ++++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 9eecc62..f429197 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -120,7 +120,7 @@ class Adaptor(SelectorsGeneration):
     def _is_text_node(element: Union[html.HtmlElement, etree._ElementUnicodeResult]) -> bool:
         """Return True if given element is a result of a string expression
         Examples:
-            Xpath -> '/text()', '/@attribute' etc...
+            XPath -> '/text()', '/@attribute' etc...
             CSS3  -> '::text', '::attr(attrib)'...
         """
         # Faster than checking `element.is_attribute or element.is_text or element.is_tail`
@@ -1007,28 +1007,25 @@ class Adaptors(List[Adaptor]):
                 return result
         return default
 
-    # def __getattr__(self, name):
-    #     if name in dir(self.__class__):
-    #         return super().__getattribute__(name)
-    #
-    #     # Execute the method itself on each Adaptor
-    #     results = []
-    #     for item in self:
-    #         results.append(getattr(item, name))
-    #
-    #     if all(callable(r) for r in results):
-    #         def call_all(*args, **kwargs):
-    #             final_results = [r(*args, **kwargs) for r in results]
-    #             if all([isinstance(r, (Adaptor, Adaptors,)) for r in results]):
-    #                 return self.__class__(final_results)
-    #             return final_results
-    #
-    #         return call_all
-    #     else:
-    #         # Flatten the result if it's a single-item list containing a list
-    #         if len(self) == 1 and isinstance(results[0], list):
-    #             return self.__class__(results[0])
-    #         return self.__class__(results)
+    def search(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptor', None]:
+        """Loop over all current elements and return the first element that matches the passed function
+        :param func: A function that takes each element as an argument and returns True/False
+        :return: The first element that match the function or ``None`` otherwise.
+        """
+        for element in self:
+            if func(element):
+                return element
+        return None
+
+    def filter(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptors', List]:
+        """Filter current elements based on the passed function
+        :param func: A function that takes each element as an argument and returns True/False
+        :return: The new `Adaptors` object or empty list otherwise.
+        """
+        results = [
+            element for element in self if func(element)
+        ]
+        return self.__class__(results) if results else results
 
     def get(self, default=None):
         """Returns the first item of the current list

From 51ce40a561e088cdbdd483aa29cad884195f9739 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 7 Nov 2024 00:44:23 +0200
Subject: [PATCH 096/118] Update ROADMAP.md

---
 ROADMAP.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ROADMAP.md b/ROADMAP.md
index ed617b7..6249bd0 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -6,7 +6,7 @@
 - [ ] Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.
 - [ ] Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
 - [ ] Need to add more functionality to `AttributesHandler` and more navigation functions to `Adaptor` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
-- [ ] Add `.filter` method to `Adaptors` object and other similar methods.
+- [x] Add `.filter` method to `Adaptors` object and other similar methods.
 - [ ] Add functionality to automatically detect pagination URLs
 - [ ] Add the ability to auto-detect schemas in pages and manipulate them.
 - [ ] Add `analyzer` ability that tries to learn about the page through meta elements and return what it learned

From b9195365274549c1d70334ec3d35ccb8e515d181 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Thu, 7 Nov 2024 12:56:32 +0200
Subject: [PATCH 097/118] Updating the keywords whitelisting logic in
 `find`/`find_all` functions

---
 scrapling/parser.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index f429197..61d4796 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -551,9 +551,10 @@ class Adaptor(SelectorsGeneration):
         """
         # Attributes that are Python reserved words and can't be used directly
         # Ex: find_all('a', class="blah") -> find_all('a', class_="blah")
+        # https://www.w3schools.com/python/python_ref_keywords.asp
         whitelisted = {
-            'id_': 'id',
             'class_': 'class',
+            'for_': 'for',
         }
 
         if not args and not kwargs:
@@ -582,14 +583,17 @@ class Adaptor(SelectorsGeneration):
 
         if not all([(type(k) is str and type(v) is str) for k, v in kwargs.items()]):
             raise TypeError('Only string values are accepted for arguments')
-        attributes.update(kwargs)
+
+        for attribute_name, value in kwargs.items():
+            # Only replace names for kwargs, replacing them in dictionaries doesn't make sense
+            attribute_name = whitelisted.get(attribute_name, attribute_name)
+            attributes[attribute_name] = value
 
         # It's easier and faster to build a selector than traversing the tree
         tags = tags or ['']
         for tag in tags:
             selector = tag
             for key, value in attributes.items():
-                key = whitelisted.get(key, key)
                 value = value.replace('"', r'\"')  # Escape double quotes in user input
                 # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
                 selector += '[{}="{}"]'.format(key, value)

From 1b015bbd073aaf5d2d52621c9243cfc793bd2da4 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Fri, 8 Nov 2024 23:59:12 +0200
Subject: [PATCH 098/118] Type correction

---
 scrapling/engines/camo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index fd43e02..272559c 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -1,5 +1,5 @@
 import logging
-from scrapling.core._types import Union, Callable, Optional, Dict, List
+from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
 
 from scrapling.engines.toolbelt import (
     Response,
@@ -15,7 +15,7 @@ from camoufox.sync_api import Camoufox
 
 class CamoufoxEngine:
     def __init__(
-            self, headless: Union[bool, str] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
+            self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
             wait_selector_state: str = 'attached', adaptor_arguments: Dict = None

From 530b54d964466e111b7be02667b47e2c40f9a218 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 00:17:41 +0200
Subject: [PATCH 099/118] Clearer doc-string

---
 scrapling/engines/camo.py | 2 +-
 scrapling/engines/pw.py   | 2 +-
 scrapling/fetchers.py     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 272559c..11f92ab 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -32,7 +32,7 @@ class CamoufoxEngine:
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
-        :param network_idle: Wait for the page to not do do any requests.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index b08195c..c2af911 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -40,7 +40,7 @@ class PlaywrightEngine:
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
-        :param network_idle: Wait for the page to not do do any requests.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 835a8e4..9d20520 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -87,7 +87,7 @@ class StealthyFetcher(BaseFetcher):
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
-        :param network_idle: Wait for the page to not do do any requests.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -144,7 +144,7 @@ class PlayWrightFetcher(BaseFetcher):
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
-        :param network_idle: Wait for the page to not do do any requests.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.

From fd34cd9afab90eb392f1c9c7481757576fe46fd3 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 00:43:24 +0200
Subject: [PATCH 100/118] Adding arguments `google_search` and `extra_headers`
 to StealthyFetcher

---
 scrapling/engines/camo.py | 17 ++++++++++++++---
 scrapling/fetchers.py     |  8 ++++++--
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 11f92ab..5335df5 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -18,14 +18,14 @@ class CamoufoxEngine:
             self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
-            wait_selector_state: str = 'attached', adaptor_arguments: Dict = None
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
 
         :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
@@ -37,6 +37,8 @@ class CamoufoxEngine:
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
         self.headless = headless
@@ -45,6 +47,8 @@ class CamoufoxEngine:
         self.block_webrtc = bool(block_webrtc)
         self.allow_webgl = bool(allow_webgl)
         self.network_idle = bool(network_idle)
+        self.google_search = bool(google_search)
+        self.extra_headers = extra_headers or {}
         self.addons = addons or []
         self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -80,7 +84,14 @@ class CamoufoxEngine:
             if self.disable_resources:
                 page.route("**/*", intercept_route)
 
-            res = page.goto(url, referer=generate_convincing_referer(url))
+            if self.extra_headers:
+                page.set_extra_http_headers(self.extra_headers)
+
+            if self.google_search:
+                res = page.goto(url, referer=generate_convincing_referer(url))
+            else:
+                res = page.goto(url)
+
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 9d20520..6f0cabb 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher):
             self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
             block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
-            wait_selector_state: str = 'attached',
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
@@ -80,7 +80,7 @@ class StealthyFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
@@ -92,6 +92,8 @@ class StealthyFetcher(BaseFetcher):
         :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
         engine = CamoufoxEngine(
@@ -107,6 +109,8 @@ class StealthyFetcher(BaseFetcher):
             network_idle=network_idle,
             wait_selector=wait_selector,
             wait_selector_state=wait_selector_state,
+            google_search=google_search,
+            extra_headers=extra_headers,
             adaptor_arguments=self.adaptor_arguments,
         )
         return engine.fetch(url)

From ee8f78feb1ac1761584580e0f1bc078ef282d391 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 00:48:11 +0200
Subject: [PATCH 101/118] Update camo.py

---
 scrapling/engines/camo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 5335df5..f56ad69 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -87,11 +87,7 @@ class CamoufoxEngine:
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
 
-            if self.google_search:
-                res = page.goto(url, referer=generate_convincing_referer(url))
-            else:
-                res = page.goto(url)
-
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:

From fd7d787ab005c5a440f8186f49ac812180f52707 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 00:54:11 +0200
Subject: [PATCH 102/118] Add `extra_headers` argument to PW and isolate google
 referer feature from stealth mode as `google_search`

---
 scrapling/engines/pw.py | 14 ++++++++++++--
 scrapling/fetchers.py   |  9 ++++++---
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index c2af911..5a71112 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -31,12 +31,14 @@ class PlaywrightEngine:
             cdp_url: Optional[str] = None,
             nstbrowser_mode: bool = False,
             nstbrowser_config: Optional[Dict] = None,
+            google_search: Optional[bool] = True,
+            extra_headers: Optional[Dict[str, str]] = None,
             adaptor_arguments: Dict = None
     ):
         """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
 
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
-        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
@@ -50,6 +52,8 @@ class PlaywrightEngine:
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
@@ -59,6 +63,8 @@ class PlaywrightEngine:
         self.stealth = bool(stealth)
         self.hide_canvas = bool(hide_canvas)
         self.disable_webgl = bool(disable_webgl)
+        self.google_search = bool(google_search)
+        self.extra_headers = extra_headers or {}
         self.cdp_url = cdp_url
         self.useragent = useragent
         self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -168,6 +174,10 @@ class PlaywrightEngine:
             page = context.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
+
+            if self.extra_headers:
+                page.set_extra_http_headers(self.extra_headers)
+
             if self.disable_resources:
                 page.route("**/*", intercept_route)
 
@@ -189,7 +199,7 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
             page.wait_for_load_state(state="load")
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 6f0cabb..4af3328 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -126,8 +126,7 @@ class PlayWrightFetcher(BaseFetcher):
             1) Patches the CDP runtime fingerprint.
             2) Mimics some of real browsers' properties by injects several JS files and using custom options.
             3) Using custom flags on launch to hide playwright even more and make it faster.
-            4) Sets the referer of every request as if this request came from Google's search of this URL's domain.
-            5) Generates real browser's headers of the same type and same user OS then append it to the request.
+            4) Generates real browser's headers of the same type and same user OS then append it to the request.
         - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
         - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
         > Note that these are the main options with PlayWright but it can be mixed together.
@@ -136,7 +135,7 @@ class PlayWrightFetcher(BaseFetcher):
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
             page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
-            hide_canvas: bool = True, disable_webgl: bool = False,
+            hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
             stealth: bool = False,
             cdp_url: Optional[str] = None,
             nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
@@ -156,6 +155,8 @@ class PlayWrightFetcher(BaseFetcher):
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument overwrites the referer set here if used together.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
@@ -170,6 +171,8 @@ class PlayWrightFetcher(BaseFetcher):
             page_action=page_action,
             hide_canvas=hide_canvas,
             network_idle=network_idle,
+            google_search=google_search,
+            extra_headers=extra_headers,
             wait_selector=wait_selector,
             disable_webgl=disable_webgl,
             nstbrowser_mode=nstbrowser_mode,

From 03091beb8b2540e65b6331e1df79900d2f09cc2a Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 13:40:50 +0200
Subject: [PATCH 103/118] Small clarification

---
 scrapling/engines/pw.py | 2 +-
 scrapling/fetchers.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 5a71112..4f453fe 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -50,7 +50,7 @@ class PlaywrightEngine:
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 4af3328..a8e9a1d 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -157,7 +157,7 @@ class PlayWrightFetcher(BaseFetcher):
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument overwrites the referer set here if used together.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.

From b90c8adb0264db13e5d524ed9ae5830a79cc5ee9 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 15:38:27 +0200
Subject: [PATCH 104/118] Adding the argument `automatch_domain` to all
 fetchers

---
 scrapling/engines/toolbelt/custom.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 9dee07f..9a1ca33 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -27,10 +27,11 @@ class Response:
     @property
     def adaptor(self) -> Union[Adaptor, None]:
         """Generate Adaptor instance from this response if possible, otherwise return None"""
+        automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
         if self.content:
-            return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
+            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
         elif self.text:
-            return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
+            return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
         return None
 
     def __repr__(self):
@@ -41,6 +42,7 @@ class BaseFetcher:
     def __init__(
             self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
             storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
+            automatch_domain: Optional[str] = None,
     ):
         """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
         are detected and passed automatically from the Fetcher based on the response for accessibility.
@@ -53,6 +55,8 @@ class BaseFetcher:
         :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
         :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
             If empty, default values will be used.
+        :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
+            Otherwise, the domain of the request is used by default.
         :param debug: Enable debug mode
         """
         # Adaptor class parameters
@@ -67,6 +71,11 @@ class BaseFetcher:
         )
         # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
         setup_basic_logging(level='debug' if debug else 'info')
+        if automatch_domain:
+            if type(automatch_domain) is not str:
+                logging.warning('[Ignored] The argument "automatch_domain" must be of string type')
+            else:
+                self.adaptor_arguments.update({'automatch_domain': automatch_domain})
 
 
 def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:

From 6b62579a7e22e6826813f1d69fc1862587a03bc2 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 21:25:17 +0200
Subject: [PATCH 105/118] Making `find_all`/`find` methods on steroids

Adding the ability to find elements by regex patterns and functions.
---
 scrapling/parser.py | 65 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 51 insertions(+), 14 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 61d4796..6530c68 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,13 +1,14 @@
 import os
+import re
+import inspect
 from difflib import SequenceMatcher
 
 from scrapling.core.translator import HTMLTranslator
 from scrapling.core.mixins import SelectorsGeneration
 from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
 from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
-from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, _is_iterable, html_forbidden
+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
 from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable
-
 from lxml import etree, html
 from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
 
@@ -542,10 +543,10 @@ class Adaptor(SelectorsGeneration):
         except (SelectorError, SelectorSyntaxError, etree.XPathError, etree.XPathEvalError):
             raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")
 
-    def find_all(self, *args, **kwargs) -> Union['Adaptors[Adaptor]', List]:
-        """Find elements by their tag name and filter them based on attributes for ease..
+    def find_all(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptors[Adaptor]', List]:
+        """Find elements by filters of your creations for ease..
 
-        :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.
+        :param args: Tag name(s), an iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
         :param kwargs: The attributes you want to filter elements based on it.
         :return: The `Adaptors` object of the elements or empty list
         """
@@ -560,9 +561,18 @@ class Adaptor(SelectorsGeneration):
         if not args and not kwargs:
             raise TypeError('You have to pass something to search with, like tag name(s), tag attributes, or both.')
 
-        tags = set()
-        selectors = []
         attributes = dict()
+        tags, patterns = set(), set()
+        results, functions, selectors = [], [], []
+
+        def _search_tree(element: Adaptor, filter_function: Callable) -> None:
+            """Collect element if it fulfills passed function otherwise, traverse the children tree and iterate"""
+            if filter_function(element):
+                results.append(element)
+
+            for branch in element.children:
+                _search_tree(branch, filter_function)
+
         # Brace yourself for a wonderful journey!
         for arg in args:
             if type(arg) is str:
@@ -578,6 +588,15 @@ class Adaptor(SelectorsGeneration):
                     raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted')
                 attributes.update(arg)
 
+            elif type(arg) is re.Pattern:
+                patterns.add(arg)
+
+            elif callable(arg):
+                if len(inspect.signature(arg).parameters) > 0:
+                    functions.append(arg)
+                else:
+                    raise TypeError("Callable filter function must have at least one argument to take `Adaptor` objects.")
+
             else:
                 raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.')
 
@@ -597,14 +616,32 @@ class Adaptor(SelectorsGeneration):
                 value = value.replace('"', r'\"')  # Escape double quotes in user input
                 # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
                 selector += '[{}="{}"]'.format(key, value)
-            selectors.append(selector)
+            if selector:
+                selectors.append(selector)
 
-        return self.css(', '.join(selectors))
+        if selectors:
+            results = self.css(', '.join(selectors))
+            if results:
+                # From the results, get the ones that fulfill passed regex patterns
+                for pattern in patterns:
+                    results = results.filter(lambda e: e.text.re(pattern, check_match=True))
 
-    def find(self, *args, **kwargs) -> Union['Adaptor', None]:
-        """Find elements by their tag name and filter them based on attributes for ease then return the first result. Otherwise return `None`.
+                # From the results, get the ones that fulfill passed functions
+                for function in functions:
+                    results = results.filter(function)
+        else:
+            for pattern in patterns:
+                results.extend(self.find_by_regex(pattern, first_match=False))
 
-        :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.
+            for function in functions:
+                _search_tree(self, function)
+
+        return self.__convert_results(results)
+
+    def find(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptor', None]:
+        """Find elements by filters of your creations for ease then return the first result. Otherwise return `None`.
+
+        :param args: Tag name(s), an iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
         :param kwargs: The attributes you want to filter elements based on it.
         :return: The `Adaptor` object of the element or `None` if the result didn't match
         """
@@ -882,10 +919,10 @@ class Adaptor(SelectorsGeneration):
         return self.__convert_results(results)
 
     def find_by_regex(
-            self, query: str, first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True
+            self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True
     ) -> Union['Adaptors[Adaptor]', 'Adaptor', List]:
         """Find elements that its text content matches the input regex pattern.
-        :param query: Regex query to match
+        :param query: Regex query/pattern to match
         :param first_match: Return first element that matches conditions, enabled by default
         :param case_sensitive: if enabled, letters case will be taken into consideration in the regex
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching

From 4b2814404aec933772f57d38c2db5c71776ba573 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 9 Nov 2024 22:32:05 +0200
Subject: [PATCH 106/118] Fixing the functions-based filtering logic

---
 scrapling/parser.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapling/parser.py b/scrapling/parser.py
index 6530c68..cac7fef 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -633,8 +633,9 @@ class Adaptor(SelectorsGeneration):
             for pattern in patterns:
                 results.extend(self.find_by_regex(pattern, first_match=False))
 
-            for function in functions:
-                _search_tree(self, function)
+            for result in (results or [self]):
+                for function in functions:
+                    _search_tree(result, function)
 
         return self.__convert_results(results)
 

From 7d2f55bd4c4d55372d82af74f9963f0cefff2aa9 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 00:09:30 +0200
Subject: [PATCH 107/118] First version of the `0.2` README

I probably forgot some features and made some mistakes here and there so I will review it again later
---
 README.md | 368 ++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 319 insertions(+), 49 deletions(-)

diff --git a/README.md b/README.md
index 84e6ada..d00de09 100644
--- a/README.md
+++ b/README.md
@@ -1,30 +1,77 @@
-# 🕷️ ScrapLing: Lightning-Fast, Adaptive Web Scraping for Python
+# 🕷️ Scrapling: Undetectable, Lightning-Fast, Adaptive Web Scraping for Python
 [![Tests](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg)](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [![PyPI version](https://badge.fury.io/py/Scrapling.svg)](https://badge.fury.io/py/Scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![PyPI Downloads](https://static.pepy.tech/badge/scrapling)](https://pepy.tech/project/scrapling)
 
-Dealing with failing web scrapers due to website changes? Meet Scrapling.
+Dealing with failing web scrapers due to anti-bot protections or website changes? Meet Scrapling.
 
 Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. Whether you're a beginner or an expert, Scrapling provides powerful features while maintaining simplicity.
 
 ```python
-from scrapling import Adaptor
-
-# Scrape data that survives website changes
-page = Adaptor(html, auto_match=True)
-products = page.css('.product', auto_save=True)
-# Later, even if selectors change:
-products = page.css('.product', auto_match=True)  # Still finds them!
+>> from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
+# Fetch websites' source under the radar!
+>> fetcher = StealthyFetcher().fetch('https://example.com', headless=True, disable_resources=True)
+>> print(fetcher.status)
+200
+>> page = fetcher.adaptor
+>> products = page.css('.product', auto_save=True)  # Scrape data that survives website design changes!
+>> # Later, if the website structure changes, pass `auto_match=True`
+>> products = page.css('.product', auto_match=True)  # and Scrapling still finds them!
 ```
 
+## Table of content
+  * [Key Features](#key-features)
+    * [Fetch websites as you prefer](#fetch-websites-as-you-prefer)
+    * [Adaptive Scraping](#adaptive-scraping)
+    * [Performance](#performance)
+    * [Developing Experience](#developing-experience)
+  * [Getting Started](#getting-started)
+  * [Parsing Performance](#parsing-performance)
+    * [Text Extraction Speed Test (5000 nested elements).](#text-extraction-speed-test-5000-nested-elements)
+    * [Extraction By Text Speed Test](#extraction-by-text-speed-test)
+  * [Installation](#installation)
+  * [Fetching Websites Features](#fetching-websites-features)
+    * [Fetcher](#fetcher)
+    * [StealthyFetcher](#stealthyfetcher)
+    * [PlayWrightFetcher](#playwrightfetcher)
+  * [Advanced Parsing Features](#advanced-parsing-features)
+    * [Smart Navigation](#smart-navigation)
+    * [Content-based Selection & Finding Similar Elements](#content-based-selection--finding-similar-elements)
+    * [Handling Structural Changes](#handling-structural-changes)
+      * [Real World Scenario](#real-world-scenario)
+    * [Find elements by filters](#find-elements-by-filters)
+    * [Is That All?](#is-that-all)
+  * [More Advanced Usage](#more-advanced-usage)
+  * [⚡ Enlightening Questions and FAQs](#-enlightening-questions-and-faqs)
+    * [How does auto-matching work?](#how-does-auto-matching-work)
+    * [How does the auto-matching work if I didn't pass a URL while initializing the Adaptor object?](#how-does-the-auto-matching-work-if-i-didnt-pass-a-url-while-initializing-the-adaptor-object)
+    * [If all things about an element can change or get removed, what are the unique properties to be saved?](#if-all-things-about-an-element-can-change-or-get-removed-what-are-the-unique-properties-to-be-saved)
+    * [I have enabled the `auto_save`/`auto_match` parameter while selecting and it got completely ignored with a warning message](#i-have-enabled-the-auto_saveauto_match-parameter-while-selecting-and-it-got-completely-ignored-with-a-warning-message)
+    * [I have done everything as the docs but the auto-matching didn't return anything, what's wrong?](#i-have-done-everything-as-the-docs-but-the-auto-matching-didnt-return-anything-whats-wrong)
+    * [Can Scrapling replace code built on top of BeautifulSoup4?](#can-scrapling-replace-code-built-on-top-of-beautifulsoup4)
+    * [Can Scrapling replace code built on top of AutoScraper?](#can-scrapling-replace-code-built-on-top-of-autoscraper)
+    * [Is Scrapling thread-safe?](#is-scrapling-thread-safe)
+  * [Sponsors](#sponsors)
+  * [Contributing](#contributing)
+  * [Disclaimer for Scrapling Project](#disclaimer-for-scrapling-project)
+  * [License](#license)
+  * [Acknowledgments](#acknowledgments)
+  * [Thanks and References](#thanks-and-references)
+  * [Known Issues](#known-issues)
+
 ## Key Features
 
+### Fetch websites as you prefer
+- **HTTP requests**: Stealthy and fast HTTP requests with `Fetcher`
+- **Stealthy fetcher**: Annoying anti-bot protection? No problem! Scrapling can bypass almost all of them with `StealthyFetcher` with default configuration!
+- **Your preferred browser**: Use your real browser with CDP, [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless, PlayWright with stealth mode, or even vanilla PlayWright -  All is possible with `PlayWrightFetcher`!
+
 ### Adaptive Scraping
 - 🔄 **Smart Element Tracking**: Locate previously identified elements after website structure changes, using an intelligent similarity system and integrated storage.
-- 🎯 **Flexible Querying**: Use CSS selectors, XPath, text search, or regex - chain them however you want!
+- 🎯 **Flexible Querying**: Use CSS selectors, XPath, Elements filters, text search, or regex - chain them however you want!
 - 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you want on the page (Ex: other products like the product you found on the page).
-- 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using its powerful features.
+- 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features.
 
 ### Performance
-- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup by up to 237x in our tests).
+- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup in parsing by up to 620x in our tests).
 - 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint.
 - ⚡ **Fast JSON serialization**: 10x faster JSON serialization than the standard json library with more options.
 
@@ -32,23 +79,18 @@ products = page.css('.product', auto_match=True)  # Still finds them!
 - 🛠️ **Powerful Navigation API**: Traverse the DOM tree easily in all directions and get the info you want (parent, ancestors, sibling, children, next/previous element, and more).
 - 🧬 **Rich Text Processing**: All strings have built-in methods for regex matching, cleaning, and more. All elements' attributes are read-only dictionaries that are faster than standard dictionaries with added methods.
 - 📝 **Automatic Selector Generation**: Create robust CSS/XPath selectors for any element.
-- 🔌 **Scrapy-Compatible API**: Familiar methods and similar pseudo-elements for Scrapy users.
-- 📘 **Type hints**: Complete type coverage for better IDE support and fewer bugs.
+- 🔌 **API Similar to Scrapy/BeautifulSoup**: Familiar methods and similar pseudo-elements for Scrapy and BeautifulSoup users.
+- 📘 **Type hints and test coverage**: Complete type coverage and almost full test coverage for better IDE support and fewer bugs, respectively.
 
 ## Getting Started
 
-Let's walk through a basic example that demonstrates a small group of Scrapling's core features:
-
 ```python
-import requests
-from scrapling import Adaptor
+from scrapling import Fetcher
 
-# Fetch a web page
-url = 'https://quotes.toscrape.com/'
-response = requests.get(url)
+fetcher = Fetcher(auto_match=False)
 
-# Create an Adaptor instance
-page = Adaptor(response.text, url=url)
+# Fetch a web page and create an Adaptor instance
+page = fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True).adaptor
 # Get all strings in the full page
 page.get_all_text(ignore_tags=('script', 'style'))
 
@@ -56,10 +98,17 @@ page.get_all_text(ignore_tags=('script', 'style'))
 quotes = page.css('.quote .text::text')  # CSS selector
 quotes = page.xpath('//span[@class="text"]/text()')  # XPath
 quotes = page.css('.quote').css('.text::text')  # Chained selectors
-quotes = [element.text for element in page.css('.quote').css('.text')]  # Slower than bulk query above
+quotes = [element.text for element in page.css('.quote .text')]  # Slower than bulk query above
 
 # Get the first quote element
-quote = page.css_first('.quote')  # or page.css('.quote').first or [0] or .get()
+quote = page.css_first('.quote')  # / page.css('.quote').first / page.css('.quote')[0]
+
+# Tired of selectors? Use find_all/find
+quotes = page.find_all('div', {'class': 'quote'})
+# Same as
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # and so on...
 
 # Working with elements
 quote.html_content  # Inner HTML
@@ -67,10 +116,9 @@ quote.prettify()  # Prettified version of Inner HTML
 quote.attrib  # Element attributes
 quote.path  # DOM path to element (List)
 ```
-To keep it simple, all methods can be chained on top of each other as long as you are chaining methods that return an element (It's called an `Adaptor` object) or a List of Adaptors (It's called `Adaptors` object)
+To keep it simple, all methods can be chained on top of each other!
 
-
-## Performance
+## Parsing Performance
 
 Scrapling isn't just powerful - it's also blazing fast. Scrapling implements many best practices, design patterns, and numerous optimizations to save fractions of seconds. All of that while focusing exclusively on parsing HTML documents.
 Here are benchmarks comparing Scrapling to popular Python libraries in two tests. 
@@ -146,7 +194,101 @@ playwright install chromium
 python -m browserforge update
 ```
 
-## Advanced Features
+## Fetching Websites Features
+All fetcher-type classes are imported with the same way
+```python
+from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
+```
+And all of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `storage`, `storage_args`, and `debug` which are the same ones you give to `Adaptor` class.
+> [!NOTE]
+> The `auto_match` argument is enabled by default which is the one you should care about the most as you will see later.
+### Fetcher
+This class is built on top of [httpx](https://www.python-httpx.org/) with some flavors, here you can do `GET`, `POST`, `PUT`, and `DELETE` requests.
+
+For all methods, you have `stealth_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default.
+```python
+>> page = Fetcher().get('https://httpbin.org/get', stealth_headers=True, follow_redirects=True)
+>> page = Fetcher().post('https://httpbin.org/post', data={'key': 'value'})
+>> page = Fetcher().put('https://httpbin.org/put', data={'key': 'value'})
+>> page = Fetcher().delete('https://httpbin.org/delete')
+```
+### StealthyFetcher
+This class is built on top of [Camoufox](https://github.com/daijro/camoufox) which is by default bypasses most of anti-bot protections. Scrapling adds extra layers of flavors and configurations to increase performance and undetectability even further.
+```python
+>> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection')  # Running headless by default
+>> page.status == 200
+True
+```
+<details><summary><strong>For the sake of simplicity, expand this for the complete list of arguments</strong></summary>
+
+|      Argument       | Description                                                                                                                                                                                                                                                                                                                                                                                                     | Optional |
+|:-------------------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
+|         url         | Target url                                                                                                                                                                                                                                                                                                                                                                                                      |    ❌     |
+|      headless       | Pass `True` to run the browser in headless/hidden (**default**), `virtual` to run it in virtual screen mode, or `False` for headful/visible mode. The `virtual` mode requires having `xvfb` installed.                                                                                                                                                                                                          |    ✔️    |
+|    block_images     | Prevent the loading of images through Firefox preferences. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._                                                                                                                                                                                                                                |    ✔️    |
+|  disable_resources  | Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._ |    ✔️    |
+|    google_search    | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.                                                                                                                                                                                                                                                                    |    ✔️    |
+|    extra_headers    | A dictionary of extra headers to add with the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._                                                                                                                                                                                                                                             |    ✔️    |
+|    block_webrtc     | Blocks WebRTC entirely.                                                                                                                                                                                                                                                                                                                                                                                         |    ✔️    |
+|     page_action     | Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.                                                                                                                                                                                                                                                                                            |    ✔️    |
+|       addons        | List of Firefox addons to use. **Must be paths to extracted addons.**                                                                                                                                                                                                                                                                                                                                           |    ✔️    |
+|      humanize       | Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.                                                                                                                                                                                                                                 |    ✔️    |
+|     allow_webgl     | Whether to allow WebGL. To prevent leaks, only use this for special cases.                                                                                                                                                                                                                                                                                                                                      |    ✔️    |
+|    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|       timeout       | The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.                                                                                                                                                                                                                                                                                                         |    ✔️    |
+|    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
+| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+
+</details>
+
+This list isn't final so expect a lot more additions and flexibility to be added in the next versions!
+
+### PlayWrightFetcher
+This class is built on top of [Playwright](https://playwright.dev/python/) which currently provides 4 main run options but they can be mixed together as you want.
+```python
+>> page = PlayWrightFetcher().fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True)  # Vanilla Playwright option
+>> page.adaptor.css_first("#search a::attr(href)")
+'https://github.com/D4Vinci/Scrapling'
+```
+Using this Fetcher class, you can do requests with:
+  1) Vanilla Playwright without any modifications other than the ones you chose.
+  2) Stealthy Playwright with the stealth mode I wrote for it. It's still a WIP but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/).</br> Some of the things this fetcher's stealth mode do includes:
+     * Patching the CDP runtime fingerprint.
+     * Mimics some of real browsers' properties by injects several JS files and using custom options.
+     * Using custom flags on launch to hide playwright even more and make it faster.
+     * Generates real browser's headers of the same type and same user OS then append it to the request's headers.
+  3) Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+  4) [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option.
+
+Add that to a lot of controlling/hiding options as you will see in the arguments list below.
+
+<details><summary><strong>Expand this for the complete list of arguments</strong></summary>
+
+|      Argument       | Description                                                                                                                                                                                                                                                                                                                                                                                                     | Optional |
+|:-------------------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
+|         url         | Target url                                                                                                                                                                                                                                                                                                                                                                                                      |    ❌     |
+|      headless       | Pass `True` to run the browser in headless/hidden (**default**), or `False` for headful/visible mode.                                                                                                                                                                                                                                                                                                           |    ✔️    |
+|  disable_resources  | Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._ |    ✔️    |
+|      useragent      | Pass a useragent string to be used. **Otherwise the fetcher will generate a real Useragent of the same browser and use it.**                                                                                                                                                                                                                                                                                    |    ✔️    |
+|    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|       timeout       | The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.                                                                                                                                                                                                                                                                                                         |    ✔️    |
+|     page_action     | Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.                                                                                                                                                                                                                                                                                            |    ✔️    |
+|    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
+| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|    google_search    | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.                                                                                                                                                                                                                                                                    |    ✔️    |
+|    extra_headers    | A dictionary of extra headers to add with the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.                                                                                                                                                                                                                                               |    ✔️    |
+|     hide_canvas     | Add random noise to canvas operations to prevent fingerprinting.                                                                                                                                                                                                                                                                                                                                                |    ✔️    |
+|    disable_webgl    | Disables WebGL and WebGL 2.0 support entirely.                                                                                                                                                                                                                                                                                                                                                                  |    ✔️    |
+|       stealth       | Enables stealth mode, always check the documentation to see what stealth mode does currently.                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|       cdp_url       | Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.                                                                                                                                                                                                                                                                                           |    ✔️    |
+|   nstbrowser_mode   | Enables NSTBrowser mode, **it have to be used with `cdp_url` argument or it will get completely ignored.**                                                                                                                                                                                                                                                                                                      |    ✔️    |
+|  nstbrowser_config  | The config you want to send with requests to the NSTBrowser. _If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config._                                                                                                                                                                                                                                                        |    ✔️    |
+
+</details>
+
+This list isn't final so expect a lot more additions and flexibility to be added in the next versions!
+
+## Advanced Parsing Features
 ### Smart Navigation
 ```python
 >>> quote.tag
@@ -166,14 +308,13 @@ python -m browserforge update
 >>> quote.siblings
 [<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
  <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
- <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
 ...]
 
 >>> quote.next  # gets the next element, the same logic applies to `quote.previous`
 <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>
 
->>> quote.children.css(".author::text")
-['Albert Einstein']
+>>> quote.children.css_first(".author::text")
+'Albert Einstein'
 
 >>> quote.has_class('quote')
 True
@@ -182,7 +323,7 @@ True
 >>> quote.css_selector
 'body > div > div:nth-of-type(2) > div > div'
 
-# Test these selectors on your favorite browser or reuse them again in the library in other methods!
+# Test these selectors on your favorite browser or reuse them again in the library's methods!
 >>> quote.xpath_selector
 '//body/div/div[2]/div/div'
 ```
@@ -200,9 +341,7 @@ You can search for a specific ancestor of an element that satisfies a function,
 ### Content-based Selection & Finding Similar Elements
 You can select elements by their text content in multiple ways, here's a full example on another website:
 ```python
->>> response = requests.get('https://books.toscrape.com/index.html')
-
->>> page = Adaptor(response.text, url=response.url)
+>>> page = Fetcher().get('https://books.toscrape.com/index.html').adaptor
 
 >>> page.find_by_text('Tipping the Velvet')  # Find the first element that its text fully matches this text
 <data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>
@@ -256,8 +395,6 @@ To increase the complexity a little bit, let's say we want to get all books' dat
 The [documentation](https://github.com/D4Vinci/Scrapling/tree/main/docs/Examples) will provide more advanced examples.
 
 ### Handling Structural Changes
-> Because [the internet archive](https://web.archive.org/) is down at the time of writing this, I can't use real websites as examples even though I tested that before (I mean browsing an old version of a website and then counting the current version of the website as structural changes)
-
 Let's say you are scraping a page with a structure like this:
 ```html
 <div class="container">
@@ -301,31 +438,137 @@ When website owners implement structural changes like
 The selector will no longer function and your code needs maintenance. That's where Scrapling auto-matching feature comes into play.
 
 ```python
+from scrapling import Adaptor
 # Before the change
-page = Adaptor(page_source, url='example.com', auto_match=True)
+page = Adaptor(page_source, url='example.com')
 element = page.css('#p1' auto_save=True)
 if not element:  # One day website changes?
-    element = page.css('#p1', auto_match=True)  # Still finds it!
+    element = page.css('#p1', auto_match=True)  # Scrapling still finds it!
 # the rest of the code...
 ```
 > How does the auto-matching work? Check the [FAQs](#-enlightening-questions-and-faqs) section for that and other possible issues while auto-matching.
 
+#### Real World Scenario
+Let's use a real website as an example and use one of the fetchers to fetch its source. To do this we need to find a website that will change its design/structure soon and take a copy of its source then wait for the website to do the change. Of course, that's nearly impossible to know unless I know website's owner but that will make it a staged test haha.
+
+To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayback Machine](https://web.archive.org/). Here is a copy of [StackOverFlow's website in 2010](https://web.archive.org/web/20100102003420/http://stackoverflow.com/), pretty old huh?</br>Let's test if the automatch feature can extract the same button in the old design from 2010 and the current design using the same selector :)
+
+If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` this selector is too specific because it was generated by Google Chrome.
+Now let's test the same selector in both versions
+```python
+>> from scrapling import Fetcher
+>> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a'
+>> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/"
+>> new_url = "https://stackoverflow.com/"
+>> 
+>> page = Fetcher(automatch_domain='stackoverflow.com').get(old_url, timeout=30).adaptor
+>> element1 = page.css_first(selector, auto_save=True)
+>> 
+>> # Same selector but used in the updated website
+>> page = Fetcher(automatch_domain="stackoverflow.com").get(new_url).adaptor
+>> element2 = page.css_first(selector, auto_match=True)
+>> 
+>> if element1.text == element2.text:
+...    print('Scrapling found the same element in the old design and the new design!')
+'Scrapling found the same element in the old design and the new design!'
+```
+Note that I used a new argument called `automatch_domain`, this because for Scrapling these are two different URLs not the website so it isolates their data. To tell Scrapling they are the same website, we the pass the domain we want to use for saving auto-match data for them both so Scrapling doesn't isolate them.
+
+In real world scenario, the code will be the same expect it will use the same URL for both requests so you won't need to use `automatch_domain` argument. This is the closest example I can give to real world cases so I hope it didn't confuse you :)
+
 **Notes:**
-1. Passing the `auto_save` argument without setting `auto_match` to `True` while initializing the Adaptor object will only result in ignoring the `auto_save` argument value and the following warning message
+1. For the two examples above I used one time the `Adaptor` class and the second time the `Fetcher` class just to show you that you can create the `Adaptor` object by yourself if you have the source or fetch the source using any `Fetcher` class then it will create the `Adaptor` object for you on the `.adaptor` property.
+2. Passing the `auto_save` argument with the `auto_match` argument set to `False` while initializing the Adaptor/Fetcher object will only result in ignoring the `auto_save` argument value and the following warning message
     ```text
     Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.
     ```
     This behavior is purely for performance reasons so the database gets created/connected only when you are planning to use the auto-matching features. Same case with the `auto_match` argument.
 
-2. The `auto_match` parameter works only for `Adaptor` instances not `Adaptors` so if you do something like this you will get an error
+3. The `auto_match` parameter works only for `Adaptor` instances not `Adaptors` so if you do something like this you will get an error
     ```python
     page.css('body').css('#p1', auto_match=True)
     ```
     because you can't auto-match a whole list, you have to be specific and do something like
     ```python
-    page.css('body')[0].css('#p1', auto_match=True)
+    page.css_first('body').css('#p1', auto_match=True)
     ```
 
+### Find elements by filters
+Inspired by BeautifulSoup's `find_all` function you can find elements by using `find_all`/`find` methods. Both methods can take multiple types of filters and returns all elements in the pages that all these filters apply to.
+
+* To be more specific:
+  * Any string passed is considered a tag name 
+  * Any iterable passed like List/Tuple/Set is considered an iterable of tag names.
+  * Any dictionary is considered a mapping of HTML element(s) attributes name and attributes values.
+  * Any regex patterns passed are used as filters
+  * Any functions passed are used as filters
+  * Any keyword argument passed is considered as a HTML element attribute with its value.
+
+So the way it works is after collecting all passed arguments and keywords, each filter passes its results to the following filter in a waterfall-like filtering system.
+<br/>It filters all elements in the current page/element in the following order:
+
+1. All elements with the passed tag name(s).
+2. All elements that matches all passed attribute(s).
+3. All elements that matches all passed regex patterns.
+4. All elements that fulfills all passed function(s).
+
+Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes passed, the process starts from that layer and so one. **But the order in which you pass the arguments doesn't matter.**
+
+Examples to clear any confusion :)
+
+```python
+>> from scrapling import Fetcher
+>> page = Fetcher().get('https://quotes.toscrape.com/').adaptor
+# Find all elements with tag name `div`.
+>> page.find_all('div')
+[<data='<div class="container"> <div class="row...' parent='<body> <div class="container"> <div clas...'>,
+ <data='<div class="row header-box"> <div class=...' parent='<div class="container"> <div class="row...'>,
+...]
+
+# Find all div elements with class that equals `quote`.
+>> page.find_all('div', class_='quote')
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+...]
+
+# Same as above.
+>> page.find_all('div', {'class': 'quote'})
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+...]
+
+# Find all elements with class that equals `quote`.
+>> page.find_all({'class': 'quote'})
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+...]
+
+# Find all div elements with class that equals `quote`, and contains the element `.text` which contains the word 'world' in its content.
+>> page.find_all('div', {'class': 'quote'}, lambda e: "world" in e.css_first('.text::text'))
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>]
+
+# Find all elements that contains the word 'world' in its content.
+>> page.find_all(lambda e: "world" in e.text)
+[<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>,
+ <data='<a class="tag" href="/tag/world/page/1/"...' parent='<div class="tags"> Tags: <meta class="ke...'>]
+
+# Find all span elements that matches the given regex
+>> page.find_all('span', re.compile(r'world'))
+[<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>]
+
+# Find all div and span elements with class 'quote' (No span elements like that so only div returned)
+>> page.find_all(['div', 'span'], {'class': 'quote'})
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+...]
+
+# Mix things up
+>> page.find_all({'itemtype':"http://schema.org/CreativeWork"}, 'div').css('.author::text')
+['Albert Einstein',
+ 'J.K. Rowling',
+...]
+```
+
 ### Is That All?
 Here's what else you can do with Scrapling:
 
@@ -355,12 +598,37 @@ Here's what else you can do with Scrapling:
     [<Element a at 0x105a2a7b0>]
     ```
 
+- Filtering results based on a function
+```python
+# Find all products over $50
+expensive_products = page.css('.product_pod').filter(
+    lambda p: float(p.css('.price_color').re_first(r'[\d\.]+')) > 50
+)
+```
+
+- Searching results for the first one that matches a function
+```python
+# Find all the product with price '53.23'
+page.css('.product_pod').search(
+    lambda p: float(p.css('.price_color').re_first(r'[\d\.]+')) == 54.23
+)
+```
+
 - Doing operations on element content is the same as scrapy
     ```python
-    quote.re(r'somethings')  # Get all strings (TextHandlers) that match the regex pattern
-    quote.re_first(r'something')  # Get the first string (TextHandler) only
+    quote.re(r'regex_pattern')  # Get all strings (TextHandlers) that match the regex pattern
+    quote.re_first(r'regex_pattern')  # Get the first string (TextHandler) only
     quote.json()  # If the content text is jsonable, then convert it to json using `orjson` which is 10x faster than the standard json library and provides more options
     ```
+    except that you can do more with them like
+    ```python
+    quote.re(
+        r'regex_pattern',
+        replace_entities=True,  # Character entity references are replaced by their corresponding character
+        clean_match=True,       # This will ignore all whitespaces and consecutive spaces while matching
+        case_sensitive= False,  # Set the regex to ignore letters case while compiling it
+    )
+    ```
     Hence all of these methods are actually methods from the `TextHandler` within that contains the text content so the same can be done directly if you call the `.text` property or equivalent selector function.
 
 
@@ -373,9 +641,9 @@ Here's what else you can do with Scrapling:
     ```python
     page.xpath('//script/text()').re_first(r'var dataLayer = (.+);').json()
     ```
-  - Sort all characters in the string as if it were a list and return the new string
+  - Sort all characters in the string as if it was a list and return the new string
     ```python
-    quote.sort()
+    quote.sort(reverse=False)
     ```
   > To be clear, `TextHandler` is a sub-class of Python's `str` so all normal operations/methods that work with Python strings will work with it.
 
@@ -406,6 +674,7 @@ There are a lot of deep details skipped here to make this as short as possible s
 
 Note that implementing your storage system can be complex as there are some strict rules such as inheriting from the same abstract class, following the singleton design pattern used in other classes, and more. So make sure to read the docs first.
 
+To give a detailed documentation of the library, it will need a website. I'm trying to rush creating the website, researching new ideas, and add more features/tests/benchmarks but time is tight with too many spinning plates between work, personal life, and working on Scrapling. But you can help by using the [sponsor button](https://github.com/sponsors/D4Vinci) above :)
 
 ## ⚡ Enlightening Questions and FAQs
 This section addresses common questions about Scrapling, please read this section before opening an issue.
@@ -460,6 +729,7 @@ Everybody is invited and welcome to contribute to Scrapling. There is a lot to d
 Please read the [contributing file](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before doing anything.
 
 ## Disclaimer for Scrapling Project
+> [!CAUTION]
 > This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international laws regarding data scraping and privacy. The authors and contributors are not responsible for any misuse of this software. This library should not be used to violate the rights of others, for unethical purposes, or to use data in an unauthorized or illegal manner. Do not use it on any website unless you have permission from the website owner or within their allowed rules like `robots.txt` file, for example.
 
 ## License
@@ -470,11 +740,11 @@ This project includes code adapted from:
 - Parsel (BSD License) - Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/translator.py) submodule
 
 ## Thanks and References
+- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
+- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright)
 - [brotector](https://github.com/kaliiiiiiiiii/brotector)
 - [fakebrowser](https://github.com/kkoooqq/fakebrowser)
 - [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches)
-- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright)
-- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
 
 ## Known Issues
 - In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone.

From 43045e3eb2fdd01e4de7a731f1d367aae03fc5ab Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 15:54:44 +0200
Subject: [PATCH 108/118] Get page content after JS Execution

Thanks for @AbdullahY36
---
 scrapling/engines/camo.py | 4 +---
 scrapling/engines/pw.py   | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index f56ad69..61731fc 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -87,9 +87,7 @@ class CamoufoxEngine:
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
-            page.wait_for_load_state(state="load")
-            page.wait_for_load_state(state="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 4f453fe..72f571a 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -199,9 +199,7 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
-            page.wait_for_load_state(state="load")
-            page.wait_for_load_state(state="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 

From 239e2c73bfe458f42b11f35cfb9ed9fe1d3a4479 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:11:46 +0200
Subject: [PATCH 109/118] Correctly get page content after JS Execution

- Thanks for @AbdullahY36 for the heads-up.

**Update reference** https://developer.mozilla.org/en-US/docs/Web/API/Document/DOMContentLoaded_event
---
 scrapling/engines/camo.py            |  5 +++--
 scrapling/engines/pw.py              |  5 +++--
 scrapling/engines/toolbelt/custom.py | 10 +++++++---
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 61731fc..6763b2b 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -87,7 +87,8 @@ class CamoufoxEngine:
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -105,7 +106,7 @@ class CamoufoxEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 72f571a..e3aa654 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -199,7 +199,8 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -217,7 +218,7 @@ class PlaywrightEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 9a1ca33..3688be2 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -28,10 +28,14 @@ class Response:
     def adaptor(self) -> Union[Adaptor, None]:
         """Generate Adaptor instance from this response if possible, otherwise return None"""
         automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
-        if self.content:
-            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
-        elif self.text:
+        if self.text:
+            # For playwright that will be the response after all JS executed
             return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.content:
+            # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
+            # To get response Bytes after the load states
+            # Reference: https://playwright.dev/python/docs/api/class-page
+            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
         return None
 
     def __repr__(self):

From de2d65856c2054cb6e0eab608a8d13bee2771b78 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:31:28 +0200
Subject: [PATCH 110/118] Adding playwright/camoufox tests for cookies and
 improve the timeout test

---
 tests/fetchers/test_camoufox.py   | 8 +++++++-
 tests/fetchers/test_playwright.py | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tests/fetchers/test_camoufox.py b/tests/fetchers/test_camoufox.py
index 0249651..0a75c09 100644
--- a/tests/fetchers/test_camoufox.py
+++ b/tests/fetchers/test_camoufox.py
@@ -15,6 +15,8 @@ class TestStealthyFetcher(unittest.TestCase):
         self.status_501 = f'{url}/status/501'
         self.basic_url = f'{url}/get'
         self.html_url = f'{url}/html'
+        self.delayed_url = f'{url}/delay/10'  # 10 Seconds delay response
+        self.cookies_url = f"{url}/cookies/set/test/value"
 
     def test_basic_fetch(self):
         """Test doing basic fetch request with multiple statuses"""
@@ -35,6 +37,10 @@ class TestStealthyFetcher(unittest.TestCase):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
 
+    def test_cookies_loading(self):
+        """Test if cookies are set after the request"""
+        self.assertEqual(self.fetcher.fetch(self.cookies_url).cookies, {'test': 'value'})
+
     def test_automation(self):
         """Test if automation break the code or not"""
         def scroll_page(page):
@@ -53,4 +59,4 @@ class TestStealthyFetcher(unittest.TestCase):
 
     def test_infinite_timeout(self):
         """Test if infinite timeout breaks the code or not"""
-        self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.delayed_url, timeout=None).status, 200)
diff --git a/tests/fetchers/test_playwright.py b/tests/fetchers/test_playwright.py
index f0ba143..138fbbe 100644
--- a/tests/fetchers/test_playwright.py
+++ b/tests/fetchers/test_playwright.py
@@ -15,6 +15,8 @@ class TestPlayWrightFetcher(unittest.TestCase):
         self.status_501 = f'{url}/status/501'
         self.basic_url = f'{url}/get'
         self.html_url = f'{url}/html'
+        self.delayed_url = f'{url}/delay/10'  # 10 Seconds delay response
+        self.cookies_url = f"{url}/cookies/set/test/value"
 
     def test_basic_fetch(self):
         """Test doing basic fetch request with multiple statuses"""
@@ -34,6 +36,10 @@ class TestPlayWrightFetcher(unittest.TestCase):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
 
+    def test_cookies_loading(self):
+        """Test if cookies are set after the request"""
+        self.assertEqual(self.fetcher.fetch(self.cookies_url).cookies, {'test': 'value'})
+
     def test_automation(self):
         """Test if automation break the code or not"""
         def scroll_page(page):
@@ -65,4 +71,4 @@ class TestPlayWrightFetcher(unittest.TestCase):
 
     def test_infinite_timeout(self):
         """Test if infinite timeout breaks the code or not"""
-        self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.delayed_url, timeout=None).status, 200)

From 3d4b427168b7a984a3b7c523306767823c9fae0e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 18:19:48 +0200
Subject: [PATCH 111/118] Rework Selectors generation

- Renamed both `css_selector` and `xpath_selector` both to `generate_css_selector` and `generate_xpath_selector` for clarity and not interrupting the auto-completion while coding.
- Added `generate_full_css_selector` and `generate_full_xpath_selector` methods
---
 scrapling/core/mixins.py     | 29 ++++++++++++++++++++++-------
 tests/parser/test_general.py |  4 ++--
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py
index e83eaa4..8b46f24 100644
--- a/scrapling/core/mixins.py
+++ b/scrapling/core/mixins.py
@@ -4,7 +4,7 @@ class SelectorsGeneration:
     Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
     Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
 
-    def __general_selection(self, selection: str = 'css') -> str:
+    def __general_selection(self, selection: str = 'css', full_path=False) -> str:
         """Generate a selector for the current element.
         :return: A string of the generated selector.
         """
@@ -20,10 +20,11 @@ class SelectorsGeneration:
                         else f"[@id='{target.attrib['id']}']"
                     )
                     selectorPath.append(part)
-                    return (
-                        " > ".join(reversed(selectorPath)) if css
-                        else '//*' + "/".join(reversed(selectorPath))
-                    )
+                    if not full_path:
+                        return (
+                            " > ".join(reversed(selectorPath)) if css
+                            else '//*' + "/".join(reversed(selectorPath))
+                        )
                 else:
                     part = f'{target.tag}'
                     # We won't use classes anymore because I some websites share exact classes between elements
@@ -60,15 +61,29 @@ class SelectorsGeneration:
         )
 
     @property
-    def css_selector(self) -> str:
+    def generate_css_selector(self) -> str:
         """Generate a CSS selector for the current element
         :return: A string of the generated selector.
         """
         return self.__general_selection()
 
     @property
-    def xpath_selector(self) -> str:
+    def generate_full_css_selector(self) -> str:
+        """Generate a complete CSS selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection(full_path=True)
+
+    @property
+    def generate_xpath_selector(self) -> str:
         """Generate a XPath selector for the current element
         :return: A string of the generated selector.
         """
         return self.__general_selection('xpath')
+
+    @property
+    def generate_full_xpath_selector(self) -> str:
+        """Generate a complete XPath selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection('xpath', full_path=True)
diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py
index 6edfced..074ad40 100644
--- a/tests/parser/test_general.py
+++ b/tests/parser/test_general.py
@@ -169,8 +169,8 @@ class TestParser(unittest.TestCase):
     def test_selectors_generation(self):
         """Try to create selectors for all elements in the page"""
         def _traverse(element: Adaptor):
-            self.assertTrue(type(element.css_selector) is str)
-            self.assertTrue(type(element.xpath_selector) is str)
+            self.assertTrue(type(element.generate_css_selector) is str)
+            self.assertTrue(type(element.generate_xpath_selector) is str)
             for branch in element.children:
                 _traverse(branch)
 

From e8935b1af811277305c14c92b2bf875b9340f866 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 23:45:48 +0200
Subject: [PATCH 112/118] Update README.md

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index d00de09..9a73fbb 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
-# 🕷️ Scrapling: Undetectable, Lightning-Fast, Adaptive Web Scraping for Python
+# 🕷️ Scrapling: Undetectable, Lightning-Fast, and Adaptive Web Scraping for Python
 [![Tests](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg)](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [![PyPI version](https://badge.fury.io/py/Scrapling.svg)](https://badge.fury.io/py/Scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![PyPI Downloads](https://static.pepy.tech/badge/scrapling)](https://pepy.tech/project/scrapling)
 
 Dealing with failing web scrapers due to anti-bot protections or website changes? Meet Scrapling.
 
-Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. Whether you're a beginner or an expert, Scrapling provides powerful features while maintaining simplicity.
+Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. For both beginners and experts, Scrapling provides powerful features while maintaining simplicity.
 
 ```python
 >> from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
@@ -203,7 +203,7 @@ And all of them can take these initialization arguments: `auto_match`, `huge_tre
 > [!NOTE]
 > The `auto_match` argument is enabled by default which is the one you should care about the most as you will see later.
 ### Fetcher
-This class is built on top of [httpx](https://www.python-httpx.org/) with some flavors, here you can do `GET`, `POST`, `PUT`, and `DELETE` requests.
+This class is built on top of [httpx](https://www.python-httpx.org/) with additional configuration options, here you can do `GET`, `POST`, `PUT`, and `DELETE` requests.
 
 For all methods, you have `stealth_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default.
 ```python
@@ -320,11 +320,11 @@ This list isn't final so expect a lot more additions and flexibility to be added
 True
 
 # Generate new selectors for any element
->>> quote.css_selector
+>>> quote.generate_css_selector
 'body > div > div:nth-of-type(2) > div > div'
 
 # Test these selectors on your favorite browser or reuse them again in the library's methods!
->>> quote.xpath_selector
+>>> quote.generate_xpath_selector
 '//body/div/div[2]/div/div'
 ```
 If your case needs more than the element's parent, you can iterate over the whole ancestors' tree of any element like below

From 79bf4060f44b62f1553009dd72397b5dab838ef8 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 12:51:40 +0200
Subject: [PATCH 113/118] Updating `find_all` examples

---
 README.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9a73fbb..6332f73 100644
--- a/README.md
+++ b/README.md
@@ -547,8 +547,15 @@ Examples to clear any confusion :)
 >> page.find_all('div', {'class': 'quote'}, lambda e: "world" in e.css_first('.text::text'))
 [<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>]
 
+# Find all elements that doesn't have children.
+>> page.find_all(lambda element: len(element.children) > 0)
+[<data='<html lang="en"><head><meta charset="UTF...'>,
+ <data='<head><meta charset="UTF-8"><title>Quote...' parent='<html lang="en"><head><meta charset="UTF...'>,
+ <data='<body> <div class="container"> <div clas...' parent='<html lang="en"><head><meta charset="UTF...'>,
+...]
+
 # Find all elements that contains the word 'world' in its content.
->> page.find_all(lambda e: "world" in e.text)
+>> page.find_all(lambda element: "world" in element.text)
 [<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>,
  <data='<a class="tag" href="/tag/world/page/1/"...' parent='<div class="tags"> Tags: <meta class="ke...'>]
 

From eb100e45f77b65cdc083705bad92e38ed7aa10ac Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:40:21 +0200
Subject: [PATCH 114/118] Corrections and rephrasing

---
 scrapling/engines/camo.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 6763b2b..3677531 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -30,15 +30,15 @@ class CamoufoxEngine:
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
-        :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
-        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
-        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
-        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
         self.headless = headless

From d6b454dc431a8d21b60ba11bc02e85983ead9cc1 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:40:30 +0200
Subject: [PATCH 115/118] Corrections and rephrasing

---
 scrapling/engines/pw.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index e3aa654..2d6ebf2 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -43,8 +43,8 @@ class PlaywrightEngine:
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
-        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
-        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -53,7 +53,7 @@ class PlaywrightEngine:
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
-        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """

From b3418f7f5036ea431a1b1ebe789893b7e5443f06 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:40:35 +0200
Subject: [PATCH 116/118] Corrections and rephrasing

---
 scrapling/engines/static.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 6c6bf84..217cd50 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -12,7 +12,7 @@ class StaticEngine:
         """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
 
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
         self.timeout = timeout

From bd38fdb347226acbe101a47b0900edabb91f716e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:40:46 +0200
Subject: [PATCH 117/118] Corrections and rephrasing

---
 scrapling/fetchers.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index a8e9a1d..65a8901 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -13,7 +13,7 @@ class Fetcher(BaseFetcher):
         """Make basic HTTP GET request for you but with some added flavors.
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
@@ -26,7 +26,7 @@ class Fetcher(BaseFetcher):
         """Make basic HTTP POST request for you but with some added flavors.
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
@@ -39,7 +39,7 @@ class Fetcher(BaseFetcher):
         """Make basic HTTP PUT request for you but with some added flavors.
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
         create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
@@ -52,7 +52,7 @@ class Fetcher(BaseFetcher):
         """Make basic HTTP DELETE request for you but with some added flavors.
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
@@ -85,15 +85,15 @@ class StealthyFetcher(BaseFetcher):
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param block_webrtc: Blocks WebRTC entirely.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
-        :param humanize: Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
         :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
-        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
-        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
-        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
         """
         engine = CamoufoxEngine(
@@ -122,10 +122,10 @@ class PlayWrightFetcher(BaseFetcher):
      Using this Fetcher class, you can do requests with:
         - Vanilla Playwright without any modifications other than the ones you chose.
         - Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com
-        Some of the things stealth mode do includes:
+        Some of the things stealth mode does include:
             1) Patches the CDP runtime fingerprint.
-            2) Mimics some of real browsers' properties by injects several JS files and using custom options.
-            3) Using custom flags on launch to hide playwright even more and make it faster.
+            2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
+            3) Using custom flags on launch to hide Playwright even more and make it faster.
             4) Generates real browser's headers of the same type and same user OS then append it to the request.
         - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
         - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
@@ -148,15 +148,15 @@ class PlayWrightFetcher(BaseFetcher):
             This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
-        :param timeout: The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.
-        :param page_action: Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
-        :param extra_headers: A dictionary of extra headers to add to headers on the request. The referer set by the `google_search` argument overwrites the referer set here if used together.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
         :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
         :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.

From 64c9fe905eb6fdb4de88e620b8ea445a6bcc0538 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:40:54 +0200
Subject: [PATCH 118/118] Corrections and rephrasing

---
 README.md | 100 +++++++++++++++++++++++++++---------------------------
 1 file changed, 50 insertions(+), 50 deletions(-)

diff --git a/README.md b/README.md
index 6332f73..bf193f4 100644
--- a/README.md
+++ b/README.md
@@ -145,7 +145,7 @@ As you see, Scrapling is on par with Scrapy and slightly faster than Lxml which
 |  Scrapling  |   2.51    |     1.0x     |
 | AutoScraper |   11.41   |    4.546x    |
 
-Scrapling can find elements with more methods and it returns full element `Adaptor` objects not only the text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them. As you see, Scrapling is still 4.5 times faster at same task.
+Scrapling can find elements with more methods and it returns full element `Adaptor` objects not only the text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them. As you see, Scrapling is still 4.5 times faster at the same task.
 
 > All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
 
@@ -154,7 +154,7 @@ Scrapling is a breeze to get started with - Starting from version 0.2, we requir
 ```bash
 pip3 install scrapling
 ```
-- For using the `StealthyFetcher`, go in the commandline and download the browser with
+- For using the `StealthyFetcher`, go to the command line and download the browser with
 <details><summary>Windows OS</summary>
 
 ```bash
@@ -185,21 +185,21 @@ On a fresh installation of Linux, you may also need the following Firefox depend
 
 <small> See the official <a href="https://camoufox.com/python/installation/#download-the-browser">Camoufox documentation</a> for more info on installation</small>
 
-- If you are going to use the `PlayWrightFetcher` options, then install playwright's chromium browser with:
+- If you are going to use the `PlayWrightFetcher` options, then install Playwright's Chromium browser with:
 ```commandline
 playwright install chromium
 ```
-- If you are going to use normal requests only with `Fetcher` class then update the fingerprints files with:
+- If you are going to use normal requests only with the `Fetcher` class then update the fingerprints files with:
 ```commandline
 python -m browserforge update
 ```
 
 ## Fetching Websites Features
-All fetcher-type classes are imported with the same way
+All fetcher-type classes are imported in the same way
 ```python
 from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
 ```
-And all of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `storage`, `storage_args`, and `debug` which are the same ones you give to `Adaptor` class.
+And all of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `storage`, `storage_args`, and `debug` which are the same ones you give to the `Adaptor` class.
 > [!NOTE]
 > The `auto_match` argument is enabled by default which is the one you should care about the most as you will see later.
 ### Fetcher
@@ -213,7 +213,7 @@ For all methods, you have `stealth_headers` which makes `Fetcher` create and use
 >> page = Fetcher().delete('https://httpbin.org/delete')
 ```
 ### StealthyFetcher
-This class is built on top of [Camoufox](https://github.com/daijro/camoufox) which is by default bypasses most of anti-bot protections. Scrapling adds extra layers of flavors and configurations to increase performance and undetectability even further.
+This class is built on top of [Camoufox](https://github.com/daijro/camoufox) which by default bypasses most of the anti-bot protections. Scrapling adds extra layers of flavors and configurations to increase performance and undetectability even further.
 ```python
 >> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection')  # Running headless by default
 >> page.status == 200
@@ -228,14 +228,14 @@ True
 |    block_images     | Prevent the loading of images through Firefox preferences. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._                                                                                                                                                                                                                                |    ✔️    |
 |  disable_resources  | Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._ |    ✔️    |
 |    google_search    | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.                                                                                                                                                                                                                                                                    |    ✔️    |
-|    extra_headers    | A dictionary of extra headers to add with the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._                                                                                                                                                                                                                                             |    ✔️    |
+|    extra_headers    | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._                                                                                                                                                                                                                                               |    ✔️    |
 |    block_webrtc     | Blocks WebRTC entirely.                                                                                                                                                                                                                                                                                                                                                                                         |    ✔️    |
-|     page_action     | Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.                                                                                                                                                                                                                                                                                            |    ✔️    |
+|     page_action     | Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.                                                                                                                                                                                                                                                                                         |    ✔️    |
 |       addons        | List of Firefox addons to use. **Must be paths to extracted addons.**                                                                                                                                                                                                                                                                                                                                           |    ✔️    |
-|      humanize       | Humanize the cursor movement. Takes either True, or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.                                                                                                                                                                                                                                 |    ✔️    |
+|      humanize       | Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.                                                                                                                                                                                                                                  |    ✔️    |
 |     allow_webgl     | Whether to allow WebGL. To prevent leaks, only use this for special cases.                                                                                                                                                                                                                                                                                                                                      |    ✔️    |
 |    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
-|       timeout       | The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.                                                                                                                                                                                                                                                                                                         |    ✔️    |
+|       timeout       | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.                                                                                                                                                                                                                                                                                                    |    ✔️    |
 |    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
 | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 
@@ -244,18 +244,18 @@ True
 This list isn't final so expect a lot more additions and flexibility to be added in the next versions!
 
 ### PlayWrightFetcher
-This class is built on top of [Playwright](https://playwright.dev/python/) which currently provides 4 main run options but they can be mixed together as you want.
+This class is built on top of [Playwright](https://playwright.dev/python/) which currently provides 4 main run options but they can be mixed as you want.
 ```python
 >> page = PlayWrightFetcher().fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True)  # Vanilla Playwright option
 >> page.adaptor.css_first("#search a::attr(href)")
 'https://github.com/D4Vinci/Scrapling'
 ```
-Using this Fetcher class, you can do requests with:
+Using this Fetcher class, you can make requests with:
   1) Vanilla Playwright without any modifications other than the ones you chose.
-  2) Stealthy Playwright with the stealth mode I wrote for it. It's still a WIP but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/).</br> Some of the things this fetcher's stealth mode do includes:
+  2) Stealthy Playwright with the stealth mode I wrote for it. It's still a WIP but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/).</br> Some of the things this fetcher's stealth mode does include:
      * Patching the CDP runtime fingerprint.
-     * Mimics some of real browsers' properties by injects several JS files and using custom options.
-     * Using custom flags on launch to hide playwright even more and make it faster.
+     * Mimics some of the real browsers' properties by injecting several JS files and using custom options.
+     * Using custom flags on launch to hide Playwright even more and make it faster.
      * Generates real browser's headers of the same type and same user OS then append it to the request's headers.
   3) Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
   4) [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option.
@@ -271,12 +271,12 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
 |  disable_resources  | Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage but be careful with this option as it makes some websites never finish loading._ |    ✔️    |
 |      useragent      | Pass a useragent string to be used. **Otherwise the fetcher will generate a real Useragent of the same browser and use it.**                                                                                                                                                                                                                                                                                    |    ✔️    |
 |    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
-|       timeout       | The timeout in milliseconds that's used in all operations and waits through the page. Default is 30000.                                                                                                                                                                                                                                                                                                         |    ✔️    |
-|     page_action     | Added for automation. A function that takes the `page` object, do the automation you need, then return `page` again.                                                                                                                                                                                                                                                                                            |    ✔️    |
+|       timeout       | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.                                                                                                                                                                                                                                                                                                    |    ✔️    |
+|     page_action     | Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.                                                                                                                                                                                                                                                                                         |    ✔️    |
 |    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
 | wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 |    google_search    | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.                                                                                                                                                                                                                                                                    |    ✔️    |
-|    extra_headers    | A dictionary of extra headers to add with the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.                                                                                                                                                                                                                                               |    ✔️    |
+|    extra_headers    | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together.                                                                                                                                                                                                                                                 |    ✔️    |
 |     hide_canvas     | Add random noise to canvas operations to prevent fingerprinting.                                                                                                                                                                                                                                                                                                                                                |    ✔️    |
 |    disable_webgl    | Disables WebGL and WebGL 2.0 support entirely.                                                                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |       stealth       | Enables stealth mode, always check the documentation to see what stealth mode does currently.                                                                                                                                                                                                                                                                                                                   |    ✔️    |
@@ -343,7 +343,7 @@ You can select elements by their text content in multiple ways, here's a full ex
 ```python
 >>> page = Fetcher().get('https://books.toscrape.com/index.html').adaptor
 
->>> page.find_by_text('Tipping the Velvet')  # Find the first element that its text fully matches this text
+>>> page.find_by_text('Tipping the Velvet')  # Find the first element whose text fully matches this text
 <data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>
 
 >>> page.find_by_text('Tipping the Velvet', first_match=False)  # Get all matches if there are more
@@ -410,7 +410,7 @@ Let's say you are scraping a page with a structure like this:
     </section>
 </div>
 ```
-and you want to scrape the first product, the one with the `p1` ID. You will probably write a selector like this
+And you want to scrape the first product, the one with the `p1` ID. You will probably write a selector like this
 ```python
 page.css('#p1')
 ```
@@ -435,7 +435,7 @@ When website owners implement structural changes like
     </div>
 </div>
 ```
-The selector will no longer function and your code needs maintenance. That's where Scrapling auto-matching feature comes into play.
+The selector will no longer function and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play.
 
 ```python
 from scrapling import Adaptor
@@ -448,12 +448,12 @@ if not element:  # One day website changes?
 ```
 > How does the auto-matching work? Check the [FAQs](#-enlightening-questions-and-faqs) section for that and other possible issues while auto-matching.
 
-#### Real World Scenario
-Let's use a real website as an example and use one of the fetchers to fetch its source. To do this we need to find a website that will change its design/structure soon and take a copy of its source then wait for the website to do the change. Of course, that's nearly impossible to know unless I know website's owner but that will make it a staged test haha.
+#### Real-World Scenario
+Let's use a real website as an example and use one of the fetchers to fetch its source. To do this we need to find a website that will change its design/structure soon, take a copy of its source then wait for the website to make the change. Of course, that's nearly impossible to know unless I know the website's owner but that will make it a staged test haha.
 
 To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayback Machine](https://web.archive.org/). Here is a copy of [StackOverFlow's website in 2010](https://web.archive.org/web/20100102003420/http://stackoverflow.com/), pretty old huh?</br>Let's test if the automatch feature can extract the same button in the old design from 2010 and the current design using the same selector :)
 
-If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` this selector is too specific because it was generated by Google Chrome.
+If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome.
 Now let's test the same selector in both versions
 ```python
 >> from scrapling import Fetcher
@@ -472,9 +472,9 @@ Now let's test the same selector in both versions
 ...    print('Scrapling found the same element in the old design and the new design!')
 'Scrapling found the same element in the old design and the new design!'
 ```
-Note that I used a new argument called `automatch_domain`, this because for Scrapling these are two different URLs not the website so it isolates their data. To tell Scrapling they are the same website, we the pass the domain we want to use for saving auto-match data for them both so Scrapling doesn't isolate them.
+Note that I used a new argument called `automatch_domain`, this is because for Scrapling these are two different URLs, not the website so it isolates their data. To tell Scrapling they are the same website, we then pass the domain we want to use for saving auto-match data for them both so Scrapling doesn't isolate them.
 
-In real world scenario, the code will be the same expect it will use the same URL for both requests so you won't need to use `automatch_domain` argument. This is the closest example I can give to real world cases so I hope it didn't confuse you :)
+In a real-world scenario, the code will be the same except it will use the same URL for both requests so you won't need to use the `automatch_domain` argument. This is the closest example I can give to real-world cases so I hope it didn't confuse you :)
 
 **Notes:**
 1. For the two examples above I used one time the `Adaptor` class and the second time the `Fetcher` class just to show you that you can create the `Adaptor` object by yourself if you have the source or fetch the source using any `Fetcher` class then it will create the `Adaptor` object for you on the `.adaptor` property.
@@ -494,25 +494,25 @@ In real world scenario, the code will be the same expect it will use the same UR
     ```
 
 ### Find elements by filters
-Inspired by BeautifulSoup's `find_all` function you can find elements by using `find_all`/`find` methods. Both methods can take multiple types of filters and returns all elements in the pages that all these filters apply to.
+Inspired by BeautifulSoup's `find_all` function you can find elements by using `find_all`/`find` methods. Both methods can take multiple types of filters and return all elements in the pages that all these filters apply to.
 
 * To be more specific:
   * Any string passed is considered a tag name 
   * Any iterable passed like List/Tuple/Set is considered an iterable of tag names.
-  * Any dictionary is considered a mapping of HTML element(s) attributes name and attributes values.
+  * Any dictionary is considered a mapping of HTML element(s) attribute names and attribute values.
   * Any regex patterns passed are used as filters
   * Any functions passed are used as filters
-  * Any keyword argument passed is considered as a HTML element attribute with its value.
+  * Any keyword argument passed is considered as an HTML element attribute with its value.
 
 So the way it works is after collecting all passed arguments and keywords, each filter passes its results to the following filter in a waterfall-like filtering system.
 <br/>It filters all elements in the current page/element in the following order:
 
 1. All elements with the passed tag name(s).
-2. All elements that matches all passed attribute(s).
-3. All elements that matches all passed regex patterns.
-4. All elements that fulfills all passed function(s).
+2. All elements that match all passed attribute(s).
+3. All elements that match all passed regex patterns.
+4. All elements that fulfill all passed function(s).
 
-Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes passed, the process starts from that layer and so one. **But the order in which you pass the arguments doesn't matter.**
+Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes are passed, the process starts from that layer and so on. **But the order in which you pass the arguments doesn't matter.**
 
 Examples to clear any confusion :)
 
@@ -525,7 +525,7 @@ Examples to clear any confusion :)
  <data='<div class="row header-box"> <div class=...' parent='<div class="container"> <div class="row...'>,
 ...]
 
-# Find all div elements with class that equals `quote`.
+# Find all div elements with a class that equals `quote`.
 >> page.find_all('div', class_='quote')
 [<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
  <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
@@ -537,29 +537,29 @@ Examples to clear any confusion :)
  <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
 ...]
 
-# Find all elements with class that equals `quote`.
+# Find all elements with a class that equals `quote`.
 >> page.find_all({'class': 'quote'})
 [<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
  <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
 ...]
 
-# Find all div elements with class that equals `quote`, and contains the element `.text` which contains the word 'world' in its content.
+# Find all div elements with a class that equals `quote`, and contains the element `.text` which contains the word 'world' in its content.
 >> page.find_all('div', {'class': 'quote'}, lambda e: "world" in e.css_first('.text::text'))
 [<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>]
 
-# Find all elements that doesn't have children.
+# Find all elements that don't have children.
 >> page.find_all(lambda element: len(element.children) > 0)
 [<data='<html lang="en"><head><meta charset="UTF...'>,
  <data='<head><meta charset="UTF-8"><title>Quote...' parent='<html lang="en"><head><meta charset="UTF...'>,
  <data='<body> <div class="container"> <div clas...' parent='<html lang="en"><head><meta charset="UTF...'>,
 ...]
 
-# Find all elements that contains the word 'world' in its content.
+# Find all elements that contain the word 'world' in its content.
 >> page.find_all(lambda element: "world" in element.text)
 [<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>,
  <data='<a class="tag" href="/tag/world/page/1/"...' parent='<div class="tags"> Tags: <meta class="ke...'>]
 
-# Find all span elements that matches the given regex
+# Find all span elements that match the given regex
 >> page.find_all('span', re.compile(r'world'))
 [<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>]
 
@@ -586,12 +586,12 @@ Here's what else you can do with Scrapling:
     ```
 - Saving and retrieving elements manually to auto-match them outside the `css` and the `xpath` methods but you have to set the identifier by yourself.
 
-  - To save element to the database:
+  - To save an element to the database:
     ```python
     >>> element = page.find_by_text('Tipping the Velvet', first_match=True)
     >>> page.save(element, 'my_special_element')
     ```
-  - Now later when you want to retrieve it and relocate it in the page with auto-matching, it would be like this
+  - Now later when you want to retrieve it and relocate it inside the page with auto-matching, it would be like this
     ```python
     >>> element_dict = page.retrieve('my_special_element')
     >>> page.relocate(element_dict, adaptor_type=True)
@@ -615,7 +615,7 @@ expensive_products = page.css('.product_pod').filter(
 
 - Searching results for the first one that matches a function
 ```python
-# Find all the product with price '53.23'
+# Find all the products with price '53.23'
 page.css('.product_pod').search(
     lambda p: float(p.css('.price_color').re_first(r'[\d\.]+')) == 54.23
 )
@@ -636,7 +636,7 @@ page.css('.product_pod').search(
         case_sensitive= False,  # Set the regex to ignore letters case while compiling it
     )
     ```
-    Hence all of these methods are actually methods from the `TextHandler` within that contains the text content so the same can be done directly if you call the `.text` property or equivalent selector function.
+    Hence all of these methods are methods from the `TextHandler` within that contains the text content so the same can be done directly if you call the `.text` property or equivalent selector function.
 
 
 - Doing operations on the text content itself includes
@@ -648,13 +648,13 @@ page.css('.product_pod').search(
     ```python
     page.xpath('//script/text()').re_first(r'var dataLayer = (.+);').json()
     ```
-  - Sort all characters in the string as if it was a list and return the new string
+  - Sort all characters in the string as if it were a list and return the new string
     ```python
     quote.sort(reverse=False)
     ```
   > To be clear, `TextHandler` is a sub-class of Python's `str` so all normal operations/methods that work with Python strings will work with it.
 
-- Any element's attributes are not exactly a dictionary but a sub-class of [mapping](https://docs.python.org/3/glossary.html#term-mapping) called `AttributesHandler` that's read-only so it's faster and string values returned are actually `TextHandler` objects so all operations above can be done on them, standard dictionary operations that doesn't modify the data, and more :)
+- Any element's attributes are not exactly a dictionary but a sub-class of [mapping](https://docs.python.org/3/glossary.html#term-mapping) called `AttributesHandler` that's read-only so it's faster and string values returned are actually `TextHandler` objects so all operations above can be done on them, standard dictionary operations that don't modify the data, and more :)
   - Unlike standard dictionaries, here you can search by values too and can do partial searches. It might be handy in some cases (returns a generator of matches)
     ```python
     >>> for item in element.attrib.search_values('catalogue', partial=True):
@@ -681,7 +681,7 @@ There are a lot of deep details skipped here to make this as short as possible s
 
 Note that implementing your storage system can be complex as there are some strict rules such as inheriting from the same abstract class, following the singleton design pattern used in other classes, and more. So make sure to read the docs first.
 
-To give a detailed documentation of the library, it will need a website. I'm trying to rush creating the website, researching new ideas, and add more features/tests/benchmarks but time is tight with too many spinning plates between work, personal life, and working on Scrapling. But you can help by using the [sponsor button](https://github.com/sponsors/D4Vinci) above :)
+To give detailed documentation of the library, it will need a website. I'm trying to rush creating the website, researching new ideas, and adding more features/tests/benchmarks but time is tight with too many spinning plates between work, personal life, and working on Scrapling. But you can help by using the [sponsor button](https://github.com/sponsors/D4Vinci) above :)
 
 ## ⚡ Enlightening Questions and FAQs
 This section addresses common questions about Scrapling, please read this section before opening an issue.
@@ -696,7 +696,7 @@ This section addresses common questions about Scrapling, please read this sectio
      Together both are used to retrieve the element's unique properties from the database later.
   4. Now later when you enable the `auto_match` parameter for both the Adaptor instance and the method call. The element properties are retrieved and Scrapling loops over all elements in the page and compares each one's unique properties to the unique properties we already have for this element and a score is calculated for each one.
   5. The comparison between elements is not exact but more about finding how similar these values are, so everything is taken into consideration even the values' order like the order in which the element class names were written before and the order in which the same element class names are written now.
-  6. The score for each element is stored in the table and in the end, the element(s) with the highest combined similarity scores are returned.
+  6. The score for each element is stored in the table, and in the end, the element(s) with the highest combined similarity scores are returned.
 
 ### How does the auto-matching work if I didn't pass a URL while initializing the Adaptor object?
 Not a big problem as it depends on your usage. The word `default` will be used in place of the URL field while saving the element's unique properties. So this will only be an issue if you used the same identifier later for a different website that you didn't pass the URL parameter while initializing it as well. The save process will overwrite the previous data and auto-matching uses the latest saved properties only.
@@ -725,7 +725,7 @@ Pretty much yeah, almost all features you get from BeautifulSoup can be found or
 Of course, you can find elements by text/regex, find similar elements in a more reliable way than AutoScraper, and finally save/retrieve elements manually to use later as the model feature in AutoScraper. I have pulled all top articles about AutoScraper from Google and tested Scrapling against examples in them. In all examples, Scrapling got the same results as AutoScraper in much less time.
 
 ### Is Scrapling thread-safe?
-Yes, Scrapling instances are thread-safe. Each Adaptor instance maintains its own state.
+Yes, Scrapling instances are thread-safe. Each Adaptor instance maintains its state.
 
 ## Sponsors
 [![Capsolver Banner](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/CapSolver.png)](https://www.capsolver.com/?utm_source=github&utm_medium=repo&utm_campaign=scraping&utm_term=Scrapling)
@@ -737,7 +737,7 @@ Please read the [contributing file](https://github.com/D4Vinci/Scrapling/blob/ma
 
 ## Disclaimer for Scrapling Project
 > [!CAUTION]
-> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international laws regarding data scraping and privacy. The authors and contributors are not responsible for any misuse of this software. This library should not be used to violate the rights of others, for unethical purposes, or to use data in an unauthorized or illegal manner. Do not use it on any website unless you have permission from the website owner or within their allowed rules like `robots.txt` file, for example.
+> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international laws regarding data scraping and privacy. The authors and contributors are not responsible for any misuse of this software. This library should not be used to violate the rights of others, for unethical purposes, or to use data in an unauthorized or illegal manner. Do not use it on any website unless you have permission from the website owner or within their allowed rules like the `robots.txt` file, for example.
 
 ## License
 This work is licensed under BSD-3