From 5f4ec3ba3a5c974a4f33f1e9752cb00f93773dea Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Feb 2025 23:14:59 +0200 Subject: [PATCH] fix(Fetcher): make `stealthy_headers` don't overwrite user headers Fix for #39 --- scrapling/engines/static.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index a939c46..a58aa39 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -42,16 +42,19 @@ class StaticEngine: :return: A dictionary of the new headers. """ headers = headers or {} - - # Validate headers - if not headers.get('user-agent') and not headers.get('User-Agent'): - headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent') - log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") + headers_keys = set(map(str.lower, headers.keys())) if self.stealth: extra_headers = generate_headers(browser_mode=False) + # Don't overwrite user supplied headers + extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} headers.update(extra_headers) - headers.update({'referer': generate_convincing_referer(self.url)}) + if 'referer' not in headers_keys: + headers.update({'referer': generate_convincing_referer(self.url)}) + + elif 'user-agent' not in headers_keys: + headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent') + log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") return headers