From b9c7a5af2e81cdc7e9acc9b73d2674a475437cc5 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 29 Jul 2025 06:08:15 +0300 Subject: [PATCH] refactor: replace's Selector inpt (text/body) with 1 argument called `content` --- benchmarks.py | 2 +- scrapling/engines/toolbelt/convertor.py | 15 ++++------ scrapling/engines/toolbelt/custom.py | 10 +++---- scrapling/parser.py | 40 +++++++++++-------------- tests/parser/test_general.py | 5 +--- 5 files changed, 30 insertions(+), 42 deletions(-) diff --git a/benchmarks.py b/benchmarks.py index 0dc451f..1ff696c 100644 --- a/benchmarks.py +++ b/benchmarks.py @@ -80,7 +80,7 @@ def test_scrapling(): @benchmark def test_parsel(): - return Selector(text=large_html).css(".item::text").extract() + return Selector(content=large_html).css(".item::text").extract() @benchmark diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 6cee891..dd0e8b1 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -34,8 +34,7 @@ class ResponseFactory: Response( url=current_request.url, # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - text="", - body=b"", + content="", status=current_response.status if current_response else 301, reason=( current_response.status_text @@ -112,8 +111,7 @@ class ResponseFactory: return Response( url=page.url, - text=page_content, - body=page_content.encode("utf-8"), + content=page_content, status=final_response.status, reason=status_text, encoding=encoding, @@ -141,8 +139,7 @@ class ResponseFactory: Response( url=current_request.url, # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - text="", - body=b"", + content="", status=current_response.status if current_response else 301, reason=( current_response.status_text @@ -221,8 +218,7 @@ class ResponseFactory: return Response( url=page.url, - text=page_content, - body=page_content.encode("utf-8"), + content=page_content, status=final_response.status, reason=status_text, encoding=encoding, @@ -243,8 +239,7 @@ class ResponseFactory: """ return Response( url=response.url, - text=response.text, - body=response.content + content=response.content if type(response.content) is bytes else response.content.encode(), status=response.status_code, diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 0416308..bc7ce15 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -103,8 +103,7 @@ class Response(Selector): def __init__( self, url: str, - text: str, - body: bytes, + content: str | bytes, status: int, reason: str, cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]], @@ -122,10 +121,11 @@ class Response(Selector): self.headers = headers self.request_headers = request_headers self.history = history or [] - encoding = ResponseEncoding.get_value(encoding, text) + encoding = ResponseEncoding.get_value( + encoding, content.decode("utf-8") if isinstance(content, bytes) else content + ) super().__init__( - text=text, - body=body, + content=content, url=adaptive_domain or url, encoding=encoding, **selector_config, diff --git a/scrapling/parser.py b/scrapling/parser.py index 41c1031..128d8d1 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -50,9 +50,8 @@ class Selector(SelectorsGeneration): def __init__( self, - text: Optional[str] = None, + content: Optional[Union[str, bytes]] = None, url: Optional[str] = None, - body: bytes = b"", encoding: str = "utf8", huge_tree: bool = True, root: Optional[html.HtmlElement] = None, @@ -72,9 +71,8 @@ class Selector(SelectorsGeneration): not possible. You can test it here and see code explodes with `AssertionError: invalid Element proxy at...`. It's an old issue with lxml, see `this entry ` - :param text: HTML body passed as text. + :param content: HTML content as either string or bytes. :param url: It allows storing a URL with the HTML data for retrieving later. - :param body: HTML body as an ``bytes`` object. It can be used instead of the ``text`` argument. :param encoding: The encoding type that will be used in HTML parsing, default is `UTF-8` :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls the libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion. @@ -88,27 +86,23 @@ class Selector(SelectorsGeneration): :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class. If empty, default values will be used. """ - if root is None and not body and text is None: + if root is None and content is None: raise ValueError( - "Selector class needs text, body, or root arguments to work" + "Selector class needs HTML content, or root arguments to work" ) self.__text = "" if root is None: - if text is None: - if not body or not isinstance(body, bytes): - raise TypeError( - f"body argument must be valid and of type bytes, got {body.__class__}" - ) - - body = body.replace(b"\x00", b"").strip() + if isinstance(content, bytes): + body = content.replace(b"\x00", b"").strip() + elif isinstance(content, str): + body = ( + content.strip().replace("\x00", "").encode(encoding) or b"" + ) else: - if not isinstance(text, str): - raise TypeError( - f"text argument must be of type str, got {text.__class__}" - ) - - body = text.strip().replace("\x00", "").encode(encoding) or b"" + raise TypeError( + f"content argument must be str or bytes, got {type(content)}" + ) # https://lxml.de/api/lxml.etree.HTMLParser-class.html parser = html.HTMLParser( @@ -122,8 +116,10 @@ class Selector(SelectorsGeneration): strip_cdata=(not keep_cdata), ) self._root = etree.fromstring(body, parser=parser, base_url=url) - if is_jsonable(text or body.decode()): - self.__text = TextHandler(text or body.decode()) + + jsonable_text = content if isinstance(content, str) else body.decode() + if is_jsonable(jsonable_text): + self.__text = TextHandler(jsonable_text) else: # All HTML types inherit from HtmlMixin so this to check for all at once @@ -930,7 +926,7 @@ class Selector(SelectorsGeneration): ) -> None: """Saves the element's unique properties to the storage for retrieval and relocation later - :param element: The element itself that we want to save to storage, it can be an ` Selector ` or pure ` HtmlElement ` + :param element: The element itself that we want to save to storage, it can be a ` Selector ` or pure ` HtmlElement ` :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See the docs for more info. """ diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 0fbac33..266e72a 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -173,10 +173,7 @@ class TestErrorHandling: _ = Selector(root="ayo", adaptive=False) with pytest.raises(TypeError): - _ = Selector(text=1, adaptive=False) - - with pytest.raises(TypeError): - _ = Selector(body=1, adaptive=False) + _ = Selector(content=1, adaptive=False) def test_invalid_storage(self, page, html_content): """Test invalid storage parameter"""