From fb80beba0443fa1909d2374169da158562e1f286 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Nov 2025 01:36:06 +0200 Subject: [PATCH] feat(TextHandler): Add argument to `clean` method to remove html entities --- scrapling/core/custom_types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 3675372..d437a23 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -103,9 +103,11 @@ class TextHandler(str): """Return a sorted version of the string""" return self.__class__("".join(sorted(self, reverse=reverse))) - def clean(self) -> Union[str, "TextHandler"]: + def clean(self, remove_entities=False) -> Union[str, "TextHandler"]: """Return a new version of the string after removing all white spaces and consecutive spaces""" data = self.translate(__CLEANING_TABLE__) + if remove_entities: + data = _replace_entities(data) return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip()) # For easy copy-paste from Scrapy/parsel code when needed :)