diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.py index d991111..d6f67b9 100644 --- a/scrapling/core/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -19,7 +19,7 @@ class StorageSystemMixin(ABC): """ self.url = url - @lru_cache(126, typed=True) + @lru_cache(64, typed=True) def _get_base_url(self, default_value: str = 'default') -> str: if not self.url or type(self.url) is not str: return default_value @@ -51,7 +51,7 @@ class StorageSystemMixin(ABC): raise NotImplementedError('Storage system must implement `save` method') @staticmethod - @lru_cache(256, typed=True) + @lru_cache(128, typed=True) def _get_hash(identifier: str) -> str: """If you want to hash identifier in your storage system, use this safer""" identifier = identifier.lower().strip() @@ -63,7 +63,7 @@ class StorageSystemMixin(ABC): return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance -@lru_cache(10, typed=True) +@lru_cache(1, typed=True) class SQLiteStorageSystem(StorageSystemMixin): """The recommended system to use, it's race condition safe and thread safe. Mainly built so the library can run in threaded frameworks like scrapy or threaded tools diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index e9de112..6555139 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -115,7 +115,7 @@ class _StorageTools: # return _impl -@lru_cache(256, typed=True) +@lru_cache(128, typed=True) def clean_spaces(string): string = string.replace('\t', ' ') string = re.sub('[\n|\r]', '', string) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 45a0ff3..0c71fec 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -126,7 +126,7 @@ class PlaywrightEngine: return cdp_url - @lru_cache(126, typed=True) + @lru_cache(32, typed=True) def __set_flags(self): """Returns the flags that will be used while launching the browser if stealth mode is enabled""" flags = DEFAULT_STEALTH_FLAGS @@ -169,7 +169,7 @@ class PlaywrightEngine: return context_kwargs - @lru_cache(10) + @lru_cache(1) def __stealth_scripts(self): # Basic bypasses nothing fancy as I'm still working on it # But with adding these bypasses to the above config, it bypasses many online tests like diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index f19b503..0aa4c2c 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache from .toolbelt import Response, generate_convincing_referer, generate_headers -@lru_cache(5, typed=True) # Singleton easily +@lru_cache(2, typed=True) # Singleton easily class StaticEngine: def __init__( self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True, diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index f00f649..c91a3a8 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -16,7 +16,7 @@ class ResponseEncoding: __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]: """Parse content type and parameters from a content-type header value. @@ -38,7 +38,7 @@ class ResponseEncoding: return content_type, params @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str: """Determine the appropriate character encoding from a content-type header. diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index cfdcf8a..dfbed5a 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union from scrapling.core.utils import lru_cache -@lru_cache(128, typed=True) +@lru_cache(10, typed=True) def generate_convincing_referer(url: str) -> str: """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website @@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str: return f'https://www.google.com/search?q={website_name}' -@lru_cache(128, typed=True) +@lru_cache(1, typed=True) def get_os_name() -> Union[str, None]: """Get the current OS name in the same format needed for browserforge diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 4d39fb5..fefb1e3 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: raise ValueError(f"Invalid CDP URL: {str(e)}") -@lru_cache(126, typed=True) +@lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it