fix: parse quoted charset values in content-type headers

`ResponseFactory.__extract_browser_encoding` matched the charset with
`charset=([\w-]+)`, which stops at a quote character. RFC 7231 permits the
charset value to be a quoted-string (e.g. `content-type: text/html;
charset="ISO-8859-1"`), so for any quoted charset the regex failed to match
and the function silently fell back to the `utf-8` default. A page served as
quoted ISO-8859-1 / windows-1252 / Shift_JIS would then be decoded as UTF-8,
producing mojibake.

Allow an optional surrounding quote in the pattern (`charset=["']?([\w-]+)`)
so the value is captured without the quote. Unquoted headers are unaffected.

The existing `content_type_map` fixture in tests/fetchers/test_utils.py was
unused; add focused tests covering unquoted, quoted, and missing charsets.
This commit is contained in:
Bortlesboat
2026-06-01 21:30:16 -04:00
parent 464fba2435
commit 6390c0af3d
2 changed files with 28 additions and 1 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ from scrapling.core.utils import log
from .custom import Response, StatusText
from scrapling.core._types import Dict, List, Optional
__CHARSET_RE__ = re_compile(r"charset=([\w-]+)")
__CHARSET_RE__ = re_compile(r"""charset=["']?([\w-]+)""")
class ResponseFactory:
+27
View File
@@ -1,5 +1,6 @@
import pytest
from scrapling.engines.toolbelt.convertor import ResponseFactory
from scrapling.engines.toolbelt.custom import StatusText, Response
from scrapling.engines.toolbelt.navigation import (
construct_proxy_dict,
@@ -139,6 +140,32 @@ def test_unknown_status_code():
assert StatusText.get(1000) == "Unknown Status Code"
# The private classmethod is name-mangled; resolve it once for the tests below.
_extract_encoding = getattr(ResponseFactory, "_ResponseFactory__extract_browser_encoding")
def test_browser_encoding_unquoted_charset():
"""A charset declared without quotes is returned verbatim."""
assert _extract_encoding("text/html; charset=utf-8") == "utf-8"
assert _extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"
assert _extract_encoding("text/html;charset=windows-1252") == "windows-1252"
def test_browser_encoding_quoted_charset():
"""A quoted charset value (RFC 7231 allows quoting) is unwrapped, not dropped."""
assert _extract_encoding('text/html; charset="utf-8"') == "utf-8"
assert _extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"
assert _extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"
assert _extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"
def test_browser_encoding_defaults_when_missing():
"""Fall back to the default when no charset is present or the header is empty."""
assert _extract_encoding("text/html") == "utf-8"
assert _extract_encoding("") == "utf-8"
assert _extract_encoding(None) == "utf-8"
class TestConstructProxyDict:
"""Test proxy dictionary construction"""