From 5ad21260305cafa1989d8737f3f15f57368046cf Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 6 Nov 2024 19:56:39 +0200 Subject: [PATCH] Small speed boost for fetching by some caching --- scrapling/engines/toolbelt/fingerprints.py | 3 +++ scrapling/engines/toolbelt/navigation.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 76f40da..71b8e84 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -4,6 +4,7 @@ Functions related to generating headers and fingerprints generally import platform +from scrapling.core.utils import cache from scrapling.core._types import Union, Dict from tldextract import extract @@ -11,6 +12,7 @@ from browserforge.headers import HeaderGenerator, Browser from browserforge.fingerprints import FingerprintGenerator, Fingerprint +@cache(None, typed=True) def generate_convincing_referer(url: str) -> str: """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website @@ -24,6 +26,7 @@ def generate_convincing_referer(url: str) -> str: return f'https://www.google.com/search?q={website_name}' +@cache(None, typed=True) def get_os_name() -> Union[str, None]: """Get the current OS name in the same format needed for browserforge diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index e03cfd7..cf73a39 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -6,6 +6,7 @@ import os import logging from urllib.parse import urlparse, urlencode +from scrapling.core.utils import cache from scrapling.core._types import Union, Dict, Optional from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES @@ -62,6 +63,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: raise ValueError(f"Invalid CDP URL: {str(e)}") +@cache(None, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it