freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: Speed up clean functions

Browse Source
This commit is contained in:
Karim shoair
2025-07-30 02:03:27 +03:00
parent 9415acccce
commit 3c5de8e0f2
2 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/custom_types.py
+4 -4
View File
@@ -18,11 +18,12 @@ from scrapling.core._types import (
Generator, Generator,
SupportsIndex, SupportsIndex,
) )
from scrapling.core.utils import _is_iterable, flatten from scrapling.core.utils import _is_iterable, flatten, __CONSECUTIVE_SPACES_REGEX__
from scrapling.core._html_utils import _replace_entities from scrapling.core._html_utils import _replace_entities
# Define type variable for AttributeHandler value type # Define type variable for AttributeHandler value type
_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler") _TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
__CLEANING_TABLE__ = str.maketrans("\t\r\n", " ")
class TextHandler(str): class TextHandler(str):
@@ -118,9 +119,8 @@ class TextHandler(str):
def clean(self) -> Union[str, "TextHandler"]: def clean(self) -> Union[str, "TextHandler"]:
"""Return a new version of the string after removing all white spaces and consecutive spaces""" """Return a new version of the string after removing all white spaces and consecutive spaces"""
trans_table = str.maketrans("\t\r\n", " ") data = self.translate(__CLEANING_TABLE__)
data = self.translate(trans_table) return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
return self.__class__(sub(" +", " ", data).strip())
# For easy copy-paste from Scrapy/parsel code when needed :) # For easy copy-paste from Scrapy/parsel code when needed :)
def get(self, default=None): def get(self, default=None):
scrapling/core/utils.py
+5 -3
View File
@@ -14,6 +14,9 @@ html_forbidden = {
html.HtmlComment, html.HtmlComment,
} }
__CLEANING_TABLE__ = str.maketrans({"\t": " ", "\n": None, "\r": None})
__CONSECUTIVE_SPACES_REGEX__ = re.compile(r" +")
@lru_cache(1, typed=True) @lru_cache(1, typed=True)
def setup_logger(): def setup_logger():
@@ -135,6 +138,5 @@ class _StorageTools:
@lru_cache(128, typed=True) @lru_cache(128, typed=True)
def clean_spaces(string): def clean_spaces(string):
string = string.replace("\t", " ") string = string.translate(__CLEANING_TABLE__)
string = re.sub("[\n|\r]", "", string) return __CONSECUTIVE_SPACES_REGEX__.sub(" ", string)
return re.sub(" +", " ", string)