freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browsers): Add option to block requests to specific domains

Browse Source
This commit is contained in:
Karim shoair
2026-02-07 01:26:21 +02:00
parent d00c34ee7f
commit 66476d42be
8 changed files with 92 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+6 -2
View File
@@ -43,6 +43,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -110,6 +111,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -138,7 +140,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
proxy = static_proxy
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page
@@ -208,6 +210,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -277,6 +280,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -306,7 +310,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
proxy = static_proxy
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page