freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix for docs

Browse Source
This commit is contained in:
Karim shoair
2024-11-23 17:48:05 +02:00
parent e51556f80c
commit 1929d9bebb
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/custom.py
+12 -12
View File
@@ -12,8 +12,8 @@ from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callab
class ResponseEncoding: class ResponseEncoding:
DEFAULT_ENCODING = "utf-8" __DEFAULT_ENCODING = "utf-8"
ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
@classmethod @classmethod
@cache(maxsize=None) @cache(maxsize=None)
@@ -43,17 +43,17 @@ class ResponseEncoding:
"""Determine the appropriate character encoding from a content-type header. """Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order: The encoding is determined by these rules in order:
1. If no content-type is provided, use UTF-8 1. If no content-type is provided, use UTF-8
2. If charset parameter is present, use that encoding 2. If charset parameter is present, use that encoding
3. If content-type is text/*, use ISO-8859-1 per HTTP/1.1 spec 3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec
4. If content-type is application/json, use UTF-8 per RFC 4627 4. If content-type is application/json, use UTF-8 per RFC 4627
5. Default to UTF-8 if nothing else matches 5. Default to UTF-8 if nothing else matches
:param content_type: Content-Type header value or None :param content_type: Content-Type header value or None
:return: String naming the character encoding :return: String naming the character encoding
""" """
if not content_type: if not content_type:
return cls.DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
try: try:
content_type, params = cls.__parse_content_type(content_type) content_type, params = cls.__parse_content_type(content_type)
@@ -65,16 +65,16 @@ class ResponseEncoding:
return encoding return encoding
# Apply content-type specific rules # Apply content-type specific rules
if content_type in cls.ISO_8859_1_CONTENT_TYPES: if content_type in cls.__ISO_8859_1_CONTENT_TYPES:
return "ISO-8859-1" return "ISO-8859-1"
if content_type == "application/json": if content_type == "application/json":
return cls.DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
return cls.DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
except (ValueError, LookupError, UnicodeEncodeError): except (ValueError, LookupError, UnicodeEncodeError):
return cls.DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
class Response(Adaptor): class Response(Adaptor):