fix: Fixes for multiple encoding issues (#80 & #81 )

scrapling/engines/toolbelt/custom.py

@@ -17,81 +17,6 @@ from scrapling.core.custom_types import MappingProxyType
 from scrapling.parser import Selector, SQLiteStorageSystem
 
 
-class ResponseEncoding:
-    __DEFAULT_ENCODING = "utf-8"
-    __ISO_8859_1_CONTENT_TYPES = {
-        "text/plain",
-        "text/html",
-        "text/css",
-        "text/javascript",
-    }
-
-    @classmethod
-    @lru_cache(maxsize=128)
-    def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
-        """Parse content type and parameters from a content-type header value.
-
-            Uses `email.message.Message` for robust header parsing according to RFC 2045.
-
-        :param header_value: Raw content-type header string
-        :return: Tuple of (content_type, parameters_dict)
-        """
-        # Create a Message object and set the Content-Type header then get the content type and parameters
-        msg = Message()
-        msg["content-type"] = header_value
-
-        content_type = msg.get_content_type()
-        params = dict(msg.get_params(failobj=[]))
-
-        # Remove the content-type from params if present somehow
-        params.pop("content-type", None)
-
-        return content_type, params
-
-    @classmethod
-    @lru_cache(maxsize=128)
-    def get_value(cls, content_type: Optional[str], text: Optional[str] = "test") -> str:
-        """Determine the appropriate character encoding from a content-type header.
-
-        The encoding is determined by these rules in order:
-            1. If no content-type is provided, use UTF-8
-            2. If charset parameter is present, use that encoding
-            3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec
-            4. If content-type is application/json, use UTF-8 per RFC 4627
-            5. Default to UTF-8 if nothing else matches
-
-        :param content_type: Content-Type header value or None
-        :param text: A text to test the encoding on it
-        :return: String naming the character encoding
-        """
-        if not content_type:
-            return cls.__DEFAULT_ENCODING
-
-        try:
-            encoding = None
-            content_type, params = cls.__parse_content_type(content_type)
-
-            # First check for explicit charset parameter
-            if "charset" in params:
-                encoding = params["charset"].strip("'\"")
-
-            # Apply content-type specific rules
-            elif content_type in cls.__ISO_8859_1_CONTENT_TYPES:
-                encoding = "ISO-8859-1"
-
-            elif content_type == "application/json":
-                encoding = cls.__DEFAULT_ENCODING
-
-            if encoding:
-                _ = text.encode(encoding)  # Validate encoding and validate it can encode the given text
-                return encoding
-
-            return cls.__DEFAULT_ENCODING
-
-        except (ValueError, LookupError, UnicodeEncodeError):
-            return cls.__DEFAULT_ENCODING
-
-
 class Response(Selector):
     """This class is returned by all engines as a way to unify response type between different libraries."""
 
@@ -116,9 +41,6 @@ class Response(Selector):
         self.headers = headers
         self.request_headers = request_headers
         self.history = history or []
-        encoding = ResponseEncoding.get_value(
-            encoding, content.decode("utf-8") if isinstance(content, bytes) else content
-        )
         super().__init__(
             content=content,
             url=adaptive_domain or url,