From c5bc82deb8134bc96db88c26a0a6fb1062422d99 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 11 Jan 2026 03:21:03 +0200 Subject: [PATCH] fix(browsers): solve a bug with setting referer on the request level --- scrapling/core/_types.py | 1 + scrapling/engines/_browsers/_controllers.py | 10 ++++++++-- scrapling/engines/_browsers/_stealth.py | 10 ++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index c897954..48a2eec 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -19,6 +19,7 @@ from typing import ( Set, Literal, Optional, + Iterator, Pattern, Sequence, Tuple, diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 285bb32..aef197a 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -108,8 +108,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin): if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") + request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None + generate_convincing_referer(url) + if (params.google_search and "referer" not in request_headers_keys) + else None ) page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) @@ -240,8 +243,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") + request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None + generate_convincing_referer(url) + if (params.google_search and "referer" not in request_headers_keys) + else None ) page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index e191059..6883fdf 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -195,8 +195,11 @@ class StealthySession(SyncSession, StealthySessionMixin): if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") + request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None + generate_convincing_referer(url) + if (params.google_search and "referer" not in request_headers_keys) + else None ) page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) @@ -410,8 +413,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") + request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None + generate_convincing_referer(url) + if (params.google_search and "referer" not in request_headers_keys) + else None ) page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)