From 6390c0af3d0a296ae895cd6a769dde8c3168fac6 Mon Sep 17 00:00:00 2001 From: Bortlesboat Date: Mon, 1 Jun 2026 21:30:16 -0400 Subject: [PATCH] fix: parse quoted charset values in content-type headers `ResponseFactory.__extract_browser_encoding` matched the charset with `charset=([\w-]+)`, which stops at a quote character. RFC 7231 permits the charset value to be a quoted-string (e.g. `content-type: text/html; charset="ISO-8859-1"`), so for any quoted charset the regex failed to match and the function silently fell back to the `utf-8` default. A page served as quoted ISO-8859-1 / windows-1252 / Shift_JIS would then be decoded as UTF-8, producing mojibake. Allow an optional surrounding quote in the pattern (`charset=["']?([\w-]+)`) so the value is captured without the quote. Unquoted headers are unaffected. The existing `content_type_map` fixture in tests/fetchers/test_utils.py was unused; add focused tests covering unquoted, quoted, and missing charsets. --- scrapling/engines/toolbelt/convertor.py | 2 +- tests/fetchers/test_utils.py | 27 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 8d624e0..cf303c8 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -10,7 +10,7 @@ from scrapling.core.utils import log from .custom import Response, StatusText from scrapling.core._types import Dict, List, Optional -__CHARSET_RE__ = re_compile(r"charset=([\w-]+)") +__CHARSET_RE__ = re_compile(r"""charset=["']?([\w-]+)""") class ResponseFactory: diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 942835f..4679478 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,5 +1,6 @@ import pytest +from scrapling.engines.toolbelt.convertor import ResponseFactory from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, @@ -139,6 +140,32 @@ def test_unknown_status_code(): assert StatusText.get(1000) == "Unknown Status Code" +# The private classmethod is name-mangled; resolve it once for the tests below. +_extract_encoding = getattr(ResponseFactory, "_ResponseFactory__extract_browser_encoding") + + +def test_browser_encoding_unquoted_charset(): + """A charset declared without quotes is returned verbatim.""" + assert _extract_encoding("text/html; charset=utf-8") == "utf-8" + assert _extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1" + assert _extract_encoding("text/html;charset=windows-1252") == "windows-1252" + + +def test_browser_encoding_quoted_charset(): + """A quoted charset value (RFC 7231 allows quoting) is unwrapped, not dropped.""" + assert _extract_encoding('text/html; charset="utf-8"') == "utf-8" + assert _extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1" + assert _extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS" + assert _extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252" + + +def test_browser_encoding_defaults_when_missing(): + """Fall back to the default when no charset is present or the header is empty.""" + assert _extract_encoding("text/html") == "utf-8" + assert _extract_encoding("") == "utf-8" + assert _extract_encoding(None) == "utf-8" + + class TestConstructProxyDict: """Test proxy dictionary construction"""