From 390bd51cb18bba5267daca5d37b31b89ff884c03 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 15:20:11 +0200
Subject: [PATCH 1/7] fix(TextHandler): correcting `case_sensitive` argument
 logic

This is awkward, I can't believe how this slipped from me!
---
 scrapling/core/custom_types.py | 22 +++++++++++-----------
 scrapling/parser.py            | 16 ++++++++--------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index 6e4ef50..30f61e1 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -134,7 +134,7 @@ class TextHandler(str):
         check_match: Literal[True],
         replace_entities: bool = True,
         clean_match: bool = False,
-        case_sensitive: bool = False,
+        case_sensitive: bool = True,
     ) -> bool:
         ...
 
@@ -144,26 +144,26 @@ class TextHandler(str):
         regex: Union[str, Pattern[str]],
         replace_entities: bool = True,
         clean_match: bool = False,
-        case_sensitive: bool = False,
+        case_sensitive: bool = True,
         check_match: Literal[False] = False,
     ) -> "TextHandlers[TextHandler]":
         ...
 
     def re(
             self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
-            case_sensitive: bool = False, check_match: bool = False
+            case_sensitive: bool = True, check_match: bool = False
     ) -> Union["TextHandlers[TextHandler]", bool]:
         """Apply the given regex to the current text and return a list of strings with the matches.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         :param check_match: used to quickly check if this regex matches or not without any operations on the results
 
         """
         if isinstance(regex, str):
-            if not case_sensitive:
+            if case_sensitive:
                 regex = re.compile(regex, re.UNICODE)
             else:
                 regex = re.compile(regex, flags=re.UNICODE | re.IGNORECASE)
@@ -182,14 +182,14 @@ class TextHandler(str):
         return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results]))
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False) -> "TextHandler":
+                 clean_match: bool = False, case_sensitive: bool = True) -> "TextHandler":
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
 
         """
         result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive)
@@ -218,14 +218,14 @@ class TextHandlers(List[TextHandler]):
         return typing.cast(_TextHandlerType, TextHandler(lst))
 
     def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
-            case_sensitive: bool = False) -> 'TextHandlers[TextHandler]':
+            case_sensitive: bool = True) -> 'TextHandlers[TextHandler]':
         """Call the ``.re()`` method for each element in this list and return
         their results flattened as TextHandlers.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         results = [
             n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
@@ -233,7 +233,7 @@ class TextHandlers(List[TextHandler]):
         return TextHandlers(flatten(results))
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False) -> TextHandler:
+                 clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
         """Call the ``.re_first()`` method for each element in this list and return
         the first result or the default value otherwise.
 
@@ -241,7 +241,7 @@ class TextHandlers(List[TextHandler]):
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         for n in self:
             for result in n.re(regex, replace_entities, clean_match, case_sensitive):
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 5a4d4f5..e6b138a 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -763,25 +763,25 @@ class Adaptor(SelectorsGeneration):
             return self.get_all_text(strip=True).json()
 
     def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True,
-           clean_match: bool = False, case_sensitive: bool = False) -> TextHandlers:
+           clean_match: bool = False, case_sensitive: bool = True) -> TextHandlers:
         """Apply the given regex to the current text and return a list of strings with the matches.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         return self.text.re(regex, replace_entities, clean_match, case_sensitive)
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False) -> TextHandler:
+                 clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         return self.text.re_first(regex, default, replace_entities, clean_match, case_sensitive)
 
@@ -1009,14 +1009,14 @@ class Adaptors(List[Adaptor]):
         return self.__class__(flatten(results))
 
     def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True,
-           clean_match: bool = False, case_sensitive: bool = False) -> TextHandlers[TextHandler]:
+           clean_match: bool = False, case_sensitive: bool = True) -> TextHandlers[TextHandler]:
         """Call the ``.re()`` method for each element in this list and return
         their results flattened as List of TextHandler.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         results = [
             n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self
@@ -1024,7 +1024,7 @@ class Adaptors(List[Adaptor]):
         return TextHandlers(flatten(results))
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False) -> TextHandler:
+                 clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
         """Call the ``.re_first()`` method for each element in this list and return
         the first result or the default value otherwise.
 
@@ -1032,7 +1032,7 @@ class Adaptors(List[Adaptor]):
         :param default: The default value to be returned if there is no match
         :param replace_entities: if enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
         """
         for n in self:
             for result in n.re(regex, replace_entities, clean_match, case_sensitive):

From f0db07a31ff14af519b3aea1f8900b5d089efc8f Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 15:25:35 +0200
Subject: [PATCH 2/7] build: pumping up deps and version to 0.2.94

---
 scrapling/__init__.py | 2 +-
 setup.cfg             | 2 +-
 setup.py              | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 7e099c8..a184e7b 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
 from scrapling.parser import Adaptor, Adaptors
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.93"
+__version__ = "0.2.94"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
diff --git a/setup.cfg b/setup.cfg
index 780469c..cb144c1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.2.93
+version = 0.2.94
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python.
diff --git a/setup.py b/setup.py
index 98a6f33..6eb2534 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="scrapling",
-    version="0.2.93",
+    version="0.2.94",
     description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It
      simplifies the process of extracting data from websites, even when they undergo structural changes, and offers
      impressive speed improvements over many popular scraping tools.""",
@@ -61,7 +61,7 @@ setup(
         'httpx[brotli,zstd, socks]',
         'playwright>=1.49.1',
         'rebrowser-playwright>=1.49.1',
-        'camoufox[geoip]>=0.4.10'
+        'camoufox[geoip]>=0.4.11'
     ],
     python_requires=">=3.9",
     url="https://github.com/D4Vinci/Scrapling",

From 3d2418b67a5d82f7f071ad2a2ce2cd2542522fb8 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 16:58:54 +0200
Subject: [PATCH 3/7] feat(Fetcher): Add the redirections history to Fetcher

Feature requested in #32
---
 scrapling/engines/static.py          | 1 +
 scrapling/engines/toolbelt/custom.py | 3 ++-
 scrapling/parser.py                  | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 9d5bed7..a939c46 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -72,6 +72,7 @@ class StaticEngine:
             headers=dict(response.headers),
             request_headers=dict(response.request.headers),
             method=response.request.method,
+            history=[self._prepare_response(redirection) for redirection in response.history],
             **self.adaptor_arguments
         )
 
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 9705eb3..dd31f4f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -85,13 +85,14 @@ class Response(Adaptor):
     """This class is returned by all engines as a way to unify response type between different libraries."""
 
     def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
-                 encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict):
+                 encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):
         automatch_domain = adaptor_arguments.pop('automatch_domain', None)
         self.status = status
         self.reason = reason
         self.cookies = cookies
         self.headers = headers
         self.request_headers = request_headers
+        self.history = history or []
         encoding = ResponseEncoding.get_value(encoding, text)
         super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
         # For back-ward compatibility
diff --git a/scrapling/parser.py b/scrapling/parser.py
index e6b138a..b83d867 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -132,7 +132,7 @@ class Adaptor(SelectorsGeneration):
         self.__tag = None
         # No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed)
         self.__response_data = {
-            key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'headers', 'request_headers',)
+            key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'history', 'headers', 'request_headers',)
         } if hasattr(self, 'status') else {}
 
     # Node functionalities, I wanted to move to separate Mixin class but it had slight impact on performance

From 8049ac0f280c0cf474635b7b6d824d6b45a80be2 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 17:47:27 +0200
Subject: [PATCH 4/7] feat(PlayWrightFetcher): Add the redirections history

---
 scrapling/engines/pw.py | 42 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index c0ca34c..6b8003c 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -259,6 +259,26 @@ class PlaywrightEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
+            history = []
+            current_request = first_response.request.redirected_from
+            while current_request:
+                current_response = current_request.response()
+
+                history.insert(0, Response(
+                    url=current_request.url,
+                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                    text='',
+                    body=b'',
+                    status=current_response.status if current_response else 301,
+                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                    cookies={},
+                    headers=current_response.all_headers() if current_response else {},
+                    request_headers=current_request.all_headers(),
+                    **self.adaptor_arguments
+                ))
+                current_request = current_request.redirected_from
+
             response = Response(
                 url=page.url,
                 text=page.content(),
@@ -269,6 +289,7 @@ class PlaywrightEngine:
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
                 headers=first_response.all_headers(),
                 request_headers=first_response.request.all_headers(),
+                history=history,
                 **self.adaptor_arguments
             )
             page.close()
@@ -345,6 +366,26 @@ class PlaywrightEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
+            history = []
+            current_request = first_response.request.redirected_from
+            while current_request:
+                current_response = await current_request.response()
+
+                history.insert(0, Response(
+                    url=current_request.url,
+                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                    text='',
+                    body=b'',
+                    status=current_response.status if current_response else 301,
+                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                    cookies={},
+                    headers=await current_response.all_headers() if current_response else {},
+                    request_headers=await current_request.all_headers(),
+                    **self.adaptor_arguments
+                ))
+                current_request = current_request.redirected_from
+
             response = Response(
                 url=page.url,
                 text=await page.content(),
@@ -355,6 +396,7 @@ class PlaywrightEngine:
                 cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
                 headers=await first_response.all_headers(),
                 request_headers=await first_response.request.all_headers(),
+                history=history,
                 **self.adaptor_arguments
             )
             await page.close()

From 10ecd414443f37ed9aa7f52ed43810d714752341 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 18:45:17 +0200
Subject: [PATCH 5/7] feat(StealthyFetcher): Add the redirections history

---
 scrapling/engines/camo.py | 42 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index e4ee4bb..8766fb1 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -142,6 +142,26 @@ class CamoufoxEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
+            history = []
+            current_request = first_response.request.redirected_from
+            while current_request:
+                current_response = current_request.response()
+
+                history.insert(0, Response(
+                    url=current_request.url,
+                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                    text='',
+                    body=b'',
+                    status=current_response.status if current_response else 301,
+                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                    cookies={},
+                    headers=current_response.all_headers() if current_response else {},
+                    request_headers=current_request.all_headers(),
+                    **self.adaptor_arguments
+                ))
+                current_request = current_request.redirected_from
+
             response = Response(
                 url=page.url,
                 text=page.content(),
@@ -152,6 +172,7 @@ class CamoufoxEngine:
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
                 headers=first_response.all_headers(),
                 request_headers=first_response.request.all_headers(),
+                history=history,
                 **self.adaptor_arguments
             )
             page.close()
@@ -223,6 +244,26 @@ class CamoufoxEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
+            history = []
+            current_request = first_response.request.redirected_from
+            while current_request:
+                current_response = await current_request.response()
+
+                history.insert(0, Response(
+                    url=current_request.url,
+                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                    text='',
+                    body=b'',
+                    status=current_response.status if current_response else 301,
+                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                    cookies={},
+                    headers=await current_response.all_headers() if current_response else {},
+                    request_headers=await current_request.all_headers(),
+                    **self.adaptor_arguments
+                ))
+                current_request = current_request.redirected_from
+
             response = Response(
                 url=page.url,
                 text=await page.content(),
@@ -233,6 +274,7 @@ class CamoufoxEngine:
                 cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
                 headers=await first_response.all_headers(),
                 request_headers=await first_response.request.all_headers(),
+                history=history,
                 **self.adaptor_arguments
             )
             await page.close()

From aa4d365901634305110e8d7d4d962a6b65f5a267 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 18:46:02 +0200
Subject: [PATCH 6/7] fix(StealthyFetcher): no need to disable coop now

Camoufox can now click inside coop iframes without it
---
 scrapling/engines/camo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 8766fb1..2f67445 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -95,7 +95,6 @@ class CamoufoxEngine:
         with Camoufox(
                 geoip=self.geoip,
                 proxy=self.proxy,
-                disable_coop=True,
                 enable_cache=True,
                 addons=self.addons,
                 exclude_addons=addons,
@@ -197,7 +196,6 @@ class CamoufoxEngine:
         async with AsyncCamoufox(
                 geoip=self.geoip,
                 proxy=self.proxy,
-                disable_coop=True,
                 enable_cache=True,
                 addons=self.addons,
                 exclude_addons=addons,

From 7f7b8e080beb3331b036bf8a64089c7f0680201e Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sat, 22 Feb 2025 18:47:02 +0200
Subject: [PATCH 7/7] docs(README): add `history` to the page to reflect new
 changes

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7c67b49..cf931f2 100644
--- a/README.md
+++ b/README.md
@@ -212,7 +212,7 @@ then use it right away without initializing like:
 page = StealthyFetcher.fetch('https://example.com') 
 ```
 
-Also, the `Response` object returned from all fetchers is the same as the `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`. All `cookies`, `headers`, and `request_headers` are always of type `dictionary`.
+Also, the `Response` object returned from all fetchers is the same as the `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, `history`, and `request_headers`. All `cookies`, `headers`, and `request_headers` are always of type `dictionary`.
 > [!NOTE]
 > The `auto_match` argument is enabled by default which is the one you should care about the most as you will see later.
 ### Fetcher