freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(fingerprints): Remove unused function and improve another

Browse Source
This commit is contained in:
Karim shoair
2025-06-15 03:16:27 +03:00
parent fcac10fa6d
commit 7efbcd33c3
2 changed files with 24 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/__init__.py
+6 -1
View File
@@ -6,7 +6,12 @@ from .custom import (
check_type_validity, check_type_validity,
get_variable_name, get_variable_name,
) )
from .fingerprints import generate_convincing_referer, generate_headers, get_os_name from .fingerprints import (
generate_convincing_referer,
generate_headers,
get_os_name,
__default_useragent__,
)
from .navigation import ( from .navigation import (
async_intercept_route, async_intercept_route,
construct_cdp_url, construct_cdp_url,
scrapling/engines/toolbelt/fingerprints.py
+18 -32
View File
@@ -4,9 +4,8 @@ Functions related to generating headers and fingerprints generally
from platform import system as platform_system from platform import system as platform_system
from browserforge.fingerprints import Fingerprint, FingerprintGenerator
from browserforge.headers import Browser, HeaderGenerator
from tldextract import extract from tldextract import extract
from browserforge.headers import Browser, HeaderGenerator
from scrapling.core._types import Dict, Union from scrapling.core._types import Dict, Union
from scrapling.core.utils import lru_cache from scrapling.core.utils import lru_cache
@@ -43,39 +42,26 @@ def get_os_name() -> Union[str, None]:
}.get(__OS_NAME__) }.get(__OS_NAME__)
def generate_suitable_fingerprint() -> Fingerprint:
"""Generates a browserforge's fingerprint that matches the current OS, desktop device, and Chrome with version 128 at least.
This function was originally created to test Browserforge's injector.
:return: `Fingerprint` object
"""
return FingerprintGenerator(
browser=[Browser(name="chrome", min_version=128)],
os=get_os_name(), # None is ignored
device="desktop",
).generate()
def generate_headers(browser_mode: bool = False) -> Dict: def generate_headers(browser_mode: bool = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator """Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
:return: A dictionary of the generated headers :return: A dictionary of the generated headers
""" """
if browser_mode: # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using, # So we don't raise any inconsistency red flags while websites fingerprinting us
# So we don't raise any inconsistency red flags while websites fingerprinting us os_name = get_os_name()
os_name = get_os_name() browsers = [Browser(name="chrome", min_version=130)]
return HeaderGenerator( if not browser_mode:
browser=[Browser(name="chrome", min_version=130)], os_name = None
os=os_name, # None is ignored browsers.extend(
device="desktop", [
).generate() Browser(name="firefox", min_version=130),
else: Browser(name="edge", min_version=130),
# Here it's used for normal requests that aren't done through browsers ]
browsers = [ )
Browser(name="chrome", min_version=130),
Browser(name="firefox", min_version=130), return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
Browser(name="edge", min_version=130),
]
return HeaderGenerator(browser=browsers, device="desktop").generate() __default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")