fix: Limit caching in all the library

To lower memory usage with high numbers of requests. Possibly improve case #48
This commit is contained in:
Karim shoair
2025-03-15 02:01:16 +02:00
parent 4f75827641
commit 7bef44ea4c
8 changed files with 13 additions and 13 deletions
+3 -3
View File
@@ -19,7 +19,7 @@ class StorageSystemMixin(ABC):
"""
self.url = url
@lru_cache(None, typed=True)
@lru_cache(126, typed=True)
def _get_base_url(self, default_value: str = 'default') -> str:
if not self.url or type(self.url) is not str:
return default_value
@@ -51,7 +51,7 @@ class StorageSystemMixin(ABC):
raise NotImplementedError('Storage system must implement `save` method')
@staticmethod
@lru_cache(None, typed=True)
@lru_cache(256, typed=True)
def _get_hash(identifier: str) -> str:
"""If you want to hash identifier in your storage system, use this safer"""
identifier = identifier.lower().strip()
@@ -63,7 +63,7 @@ class StorageSystemMixin(ABC):
return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance
@lru_cache(None, typed=True)
@lru_cache(10, typed=True)
class SQLiteStorageSystem(StorageSystemMixin):
"""The recommended system to use, it's race condition safe and thread safe.
Mainly built so the library can run in threaded frameworks like scrapy or threaded tools