From c8c494e73f09ff44bf00b4c0412ce07eb218e482 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 3 Nov 2024 16:25:55 +0200 Subject: [PATCH] Adding a function to intercept and drop unwanted requests --- scrapling/engines/toolbelt/__init__.py | 1 + scrapling/engines/toolbelt/navigation.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index 3aa0aed..375e304 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -14,5 +14,6 @@ from .custom import ( ) from .navigation import ( js_bypass_path, + intercept_route, construct_websocket_url, ) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index bf99cb2..5a8192b 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -3,7 +3,18 @@ Functions related to files and URLs """ import os +import logging from urllib.parse import urlparse, urlencode +from playwright.sync_api import Route + +from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES + + +def intercept_route(route: Route): + if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: + logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') + return route.abort() + return route.continue_() def construct_websocket_url(base_url, query_params):