From 05f62fba818679ee79d7509536ce44da792362d7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 6 Feb 2026 03:14:23 +0200 Subject: [PATCH] fix(response): Force the body of the response to always be bytes --- scrapling/engines/toolbelt/convertor.py | 8 ++++---- scrapling/engines/toolbelt/custom.py | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 9dff73d..4b4caa2 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -117,12 +117,12 @@ class ResponseFactory: history = cls._process_response_history(first_response, parser_arguments) try: if "html" in final_response.all_headers().get("content-type", ""): - page_content = cls._get_page_content(page) + page_content = cls._get_page_content(page).encode("utf-8") else: page_content = final_response.body() except Exception as e: # pragma: no cover log.error(f"Error getting page content: {e}") - page_content = "" + page_content = b"" return Response( **{ @@ -254,12 +254,12 @@ class ResponseFactory: history = await cls._async_process_response_history(first_response, parser_arguments) try: if "html" in (await final_response.all_headers()).get("content-type", ""): - page_content = await cls._get_async_page_content(page) + page_content = (await cls._get_async_page_content(page)).encode("utf-8") else: page_content = await final_response.body() except Exception as e: # pragma: no cover log.error(f"Error getting page content in async: {e}") - page_content = "" + page_content = b"" return Response( **{ diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 29b9e17..58a7485 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -14,6 +14,7 @@ from scrapling.core._types import ( Union, Optional, Callable, + Sequence, TYPE_CHECKING, AsyncGenerator, ) @@ -42,6 +43,9 @@ class Response(Selector): meta: Dict[str, Any] | None = None, **selector_config: Any, ): + if isinstance(content, str): + content = content.encode("utf-8") + adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", "")) self.status = status self.reason = reason @@ -64,6 +68,11 @@ class Response(Selector): self.meta: Dict[str, Any] = meta or {} self.request: Optional["Request"] = None # Will be set by crawler + @property + def body(self) -> bytes: + """Return the raw body of the response as bytes.""" + return cast(bytes, cast(Sequence, self._raw_body)) + def follow( self, url: str,