fix(response): Force the body of the response to always be bytes

scrapling/engines/toolbelt/convertor.py

@@ -117,12 +117,12 @@ class ResponseFactory:
         history = cls._process_response_history(first_response, parser_arguments)
         try:
             if "html" in final_response.all_headers().get("content-type", ""):
-                page_content = cls._get_page_content(page)
+                page_content = cls._get_page_content(page).encode("utf-8")
             else:
                 page_content = final_response.body()
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content: {e}")
-            page_content = ""
+            page_content = b""
 
         return Response(
             **{
@@ -254,12 +254,12 @@ class ResponseFactory:
         history = await cls._async_process_response_history(first_response, parser_arguments)
         try:
             if "html" in (await final_response.all_headers()).get("content-type", ""):
-                page_content = await cls._get_async_page_content(page)
+                page_content = (await cls._get_async_page_content(page)).encode("utf-8")
             else:
                 page_content = await final_response.body()
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content in async: {e}")
-            page_content = ""
+            page_content = b""
 
         return Response(
             **{

scrapling/engines/toolbelt/custom.py

@@ -14,6 +14,7 @@ from scrapling.core._types import (
     Union,
     Optional,
     Callable,
+    Sequence,
     TYPE_CHECKING,
     AsyncGenerator,
 )
@@ -42,6 +43,9 @@ class Response(Selector):
         meta: Dict[str, Any] | None = None,
         **selector_config: Any,
     ):
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+
         adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))
         self.status = status
         self.reason = reason
@@ -64,6 +68,11 @@ class Response(Selector):
         self.meta: Dict[str, Any] = meta or {}
         self.request: Optional["Request"] = None  # Will be set by crawler
 
+    @property
+    def body(self) -> bytes:
+        """Return the raw body of the response as bytes."""
+        return cast(bytes, cast(Sequence, self._raw_body))
+
     def follow(
         self,
         url: str,