From 72e26c23e833bc425969f0747cb8f6f76ccad40d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 03:13:51 +0300 Subject: [PATCH 01/35] fix(DynamicFetcher): Improve stealth mode --- scrapling/engines/constants.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py index c7bdb1b..03a678e 100644 --- a/scrapling/engines/constants.py +++ b/scrapling/engines/constants.py @@ -16,9 +16,9 @@ HARMFUL_DEFAULT_ARGS = ( # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884 "--enable-automation", "--disable-popup-blocking", - # '--disable-component-update', - # '--disable-default-apps', - # '--disable-extensions', + "--disable-component-update", + "--disable-default-apps", + "--disable-extensions", ) DEFAULT_FLAGS = ( @@ -50,7 +50,6 @@ DEFAULT_STEALTH_FLAGS = ( "--accept-lang=en-US", "--use-mock-keychain", "--disable-translate", - "--disable-extensions", "--disable-voice-input", "--window-position=0,0", "--disable-wake-on-wifi", @@ -59,7 +58,6 @@ DEFAULT_STEALTH_FLAGS = ( "--enable-web-bluetooth", "--disable-hang-monitor", "--disable-cloud-import", - "--disable-default-apps", "--disable-print-preview", "--disable-dev-shm-usage", # '--disable-popup-blocking', @@ -72,7 +70,6 @@ DEFAULT_STEALTH_FLAGS = ( "--force-color-profile=srgb", "--font-render-hinting=none", "--aggressive-cache-discard", - "--disable-component-update", "--disable-cookie-encryption", "--disable-domain-reliability", "--disable-threaded-animation", From 9e5ff5bb1b2d1720c97a3a282a8ffd2f726e7cd2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 04:36:00 +0300 Subject: [PATCH 02/35] chore: Add issue template for other issues --- .github/ISSUE_TEMPLATE/03-other.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/03-other.yml diff --git a/.github/ISSUE_TEMPLATE/03-other.yml b/.github/ISSUE_TEMPLATE/03-other.yml new file mode 100644 index 0000000..697c1f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/03-other.yml @@ -0,0 +1,19 @@ +name: Other +description: Use this for any other issues. PLEASE provide as much information as possible. +labels: ["awaiting triage"] +body: + - type: textarea + id: issuedescription + attributes: + label: What would you like to share? + description: Provide a clear and concise explanation of your issue. + validations: + required: true + + - type: textarea + id: extrainfo + attributes: + label: Additional information + description: Is there anything else we should know about this issue? + validations: + required: false \ No newline at end of file From 4341477a668ea2997d1f39667ed477436806a806 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 04:58:55 +0300 Subject: [PATCH 03/35] perf(StealthyFetcher): Slight optimization --- scrapling/engines/_browsers/_camoufox.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index d6150e4..44d8735 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -3,7 +3,10 @@ from re import compile as re_compile from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons -from camoufox.utils import launch_options as generate_launch_options +from camoufox.utils import ( + launch_options as generate_launch_options, + installed_verstr as camoufox_version, +) from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, @@ -41,6 +44,7 @@ from scrapling.engines.toolbelt import ( ) __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +__ff_version_str__ = camoufox_version().split(".", 1)[0] class StealthySession: @@ -216,7 +220,6 @@ class StealthySession: **{ "geoip": self.geoip, "proxy": dict(self.proxy) if self.proxy else self.proxy, - "enable_cache": True, "addons": self.addons, "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], "headless": self.headless, @@ -227,6 +230,15 @@ class StealthySession: "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. "os": None if self.os_randomize else get_os_name(), "user_data_dir": "", + "ff_version": __ff_version_str__, + "firefox_user_prefs": { + # This is what enabling `enable_cache` does internally, so we do it from here instead + "browser.sessionhistory.max_entries": 10, + "browser.sessionhistory.max_total_viewers": -1, + "browser.cache.memory.enable": True, + "browser.cache.disk_cache_ssl": True, + "browser.cache.disk.smart_size.enabled": True, + }, **self.additional_args, } ) From 05761d40700049590565276c1b2e68dfd6b17abf Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 05:06:35 +0300 Subject: [PATCH 04/35] docs: Add YT video to the mcp page --- docs/ai/mcp-server.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md index ceef23a..19088ec 100644 --- a/docs/ai/mcp-server.md +++ b/docs/ai/mcp-server.md @@ -1,5 +1,7 @@ # Scrapling MCP Server Guide + + The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful Web Scraping capabilities directly to your favorite AI chatbot or AI agent. This integration allows you to scrape websites, extract data, and bypass anti-bot protections conversationally through Claude's AI interface or any other chatbot that supports MCP. ## Features From 710f994339df127d3900e07dad845fade850e329 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 18:48:40 +0300 Subject: [PATCH 05/35] perf(DynamicSession): Close persistent context default page --- scrapling/engines/_browsers/_controllers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 2747ed9..85db198 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -248,6 +248,10 @@ class DynamicSession: user_data_dir="", **self.launch_options ) + # Get the default page and close it + default_page = self.context.pages[0] + default_page.close() + if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -505,6 +509,10 @@ class AsyncDynamicSession(DynamicSession): ) ) + # Get the default page and close it + default_page = self.context.pages[0] + await default_page.close() + if self.init_script: # pragma: no cover await self.context.add_init_script(path=self.init_script) From f298b76b8e80fda1e98311e486634ccdb99356a0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Sep 2025 18:48:51 +0300 Subject: [PATCH 06/35] perf(StealthySession): Close persistent context default page --- scrapling/engines/_browsers/_camoufox.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 44d8735..5afccb1 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -251,6 +251,11 @@ class StealthySession: **self.launch_options ) ) + + # Get the default page and close it + default_page = self.context.pages[0] + default_page.close() + if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -580,6 +585,11 @@ class AsyncStealthySession(StealthySession): **self.launch_options ) ) + + # Get the default page and close it + default_page = self.context.pages[0] + await default_page.close() + if self.init_script: # pragma: no cover await self.context.add_init_script(path=self.init_script) From ecaec65049598476ffbe15c05790eca58a04b3ae Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 8 Sep 2025 16:08:57 +0300 Subject: [PATCH 07/35] refactor/fix(browser fetchers): Solving a logic bug in page rotation - Pages that get reused in rotation are possibly contaminated from previous settings used on them, and some are stubborn to remove, so the new approach replaces finished pages with new ones. - Removed the `max_pages` from sync `StealthySession` to match `DynamicSession` (Doesn't mean anything in sync code) --- scrapling/engines/_browsers/_base.py | 403 ++++++++++++++++++++ scrapling/engines/_browsers/_camoufox.py | 269 ++----------- scrapling/engines/_browsers/_controllers.py | 249 ++---------- scrapling/engines/_browsers/_page.py | 54 ++- scrapling/fetchers.py | 1 - 5 files changed, 521 insertions(+), 455 deletions(-) create mode 100644 scrapling/engines/_browsers/_base.py diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py new file mode 100644 index 0000000..55b3310 --- /dev/null +++ b/scrapling/engines/_browsers/_base.py @@ -0,0 +1,403 @@ +from time import time, sleep +from asyncio import sleep as asyncio_sleep, Lock + +from camoufox import DefaultAddons +from playwright.sync_api import BrowserContext, Playwright +from playwright.async_api import ( + BrowserContext as AsyncBrowserContext, + Playwright as AsyncPlaywright, +) +from camoufox.utils import ( + launch_options as generate_launch_options, + installed_verstr as camoufox_version, +) + +from scrapling.engines.toolbelt import ( + intercept_route, + async_intercept_route, + get_os_name, +) +from ._page import PageInfo, PagePool +from ._config_tools import _compiled_stealth_scripts +from ._validators import validate, PlaywrightConfig, CamoufoxConfig +from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.core._types import ( + Dict, + Optional, +) + +__ff_version_str__ = camoufox_version().split(".", 1)[0] + + +class SyncSession: + def __init__(self, max_pages: int = 1): + self.max_pages = max_pages + self.page_pool = PagePool(max_pages) + self.__max_wait_for_page = 60 + self.playwright: Optional[Playwright] = None + self.context: Optional[BrowserContext] = None + self._closed = False + + def _get_page(self) -> PageInfo: # pragma: no cover + """Get a new page to use""" + + # Close all finished pages to ensure clean state + self.page_pool.close_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + sleep(0.05) + self.page_pool.close_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = self.context.new_page() + timeout = getattr(self, "timeout", 30000) + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if getattr(self, "extra_headers", False): + page.set_extra_http_headers(getattr(self, "extra_headers")) + + if getattr(self, "disable_resources", False): + page.route("**/*", intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + def get_pool_stats(self) -> Dict[str, int]: + """Get statistics about the current page pool""" + return { + "total_pages": self.page_pool.pages_count, + "busy_pages": self.page_pool.busy_count, + "max_pages": self.max_pages, + } + + +class AsyncSession(SyncSession): + def __init__(self, max_pages: int = 1): + super().__init__(max_pages) + self.playwright: Optional[AsyncPlaywright] = None + self.context: Optional[AsyncBrowserContext] = None + self._lock = Lock() + + async def _get_page(self) -> PageInfo: # pragma: no cover + """Get a new page to use""" + async with self._lock: + # Close all finished pages to ensure clean state + await self.page_pool.aclose_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + await asyncio_sleep(0.05) + await self.page_pool.aclose_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = await self.context.new_page() + timeout = getattr(self, "timeout", 30000) + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if getattr(self, "extra_headers", False): + await page.set_extra_http_headers(getattr(self, "extra_headers")) + + if getattr(self, "disable_resources", False): + await page.route("**/*", async_intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + await page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + +class DynamicSessionMixin: + def __validate__( + self, + __max_pages, + headless, + google_search, + hide_canvas, + disable_webgl, + real_chrome, + stealth, + wait, + page_action, + proxy, + locale, + extra_headers, + useragent, + cdp_url, + timeout, + disable_resources, + wait_selector, + init_script, + cookies, + network_idle, + wait_selector_state, + selector_config, + ): + params = { + "max_pages": __max_pages, + "headless": headless, + "google_search": google_search, + "hide_canvas": hide_canvas, + "disable_webgl": disable_webgl, + "real_chrome": real_chrome, + "stealth": stealth, + "wait": wait, + "page_action": page_action, + "proxy": proxy, + "locale": locale, + "extra_headers": extra_headers, + "useragent": useragent, + "timeout": timeout, + "selector_config": selector_config, + "disable_resources": disable_resources, + "wait_selector": wait_selector, + "init_script": init_script, + "cookies": cookies, + "network_idle": network_idle, + "wait_selector_state": wait_selector_state, + "cdp_url": cdp_url, + } + config = validate(params, PlaywrightConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.hide_canvas = config.hide_canvas + self.disable_webgl = config.disable_webgl + self.real_chrome = config.real_chrome + self.stealth = config.stealth + self.google_search = config.google_search + self.wait = config.wait + self.proxy = config.proxy + self.locale = config.locale + self.extra_headers = config.extra_headers + self.useragent = config.useragent + self.timeout = config.timeout + self.cookies = config.cookies + self.disable_resources = config.disable_resources + self.cdp_url = config.cdp_url + self.network_idle = config.network_idle + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.wait_selector_state = config.wait_selector_state + self.selector_config = config.selector_config + self.page_action = config.page_action + self._headers_keys = ( + set(map(str.lower, self.extra_headers.keys())) + if self.extra_headers + else set() + ) + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + if not self.cdp_url: + # `launch_options` is used with persistent context + self.launch_options = dict( + _launch_kwargs( + self.headless, + self.proxy, + self.locale, + tuple(self.extra_headers.items()) + if self.extra_headers + else tuple(), + self.useragent, + self.real_chrome, + self.stealth, + self.hide_canvas, + self.disable_webgl, + ) + ) + self.launch_options["extra_http_headers"] = dict( + self.launch_options["extra_http_headers"] + ) + self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + self.context_options = dict() + else: + # while `context_options` is left to be used when cdp mode is enabled + self.launch_options = dict() + self.context_options = dict( + _context_kwargs( + self.proxy, + self.locale, + tuple(self.extra_headers.items()) + if self.extra_headers + else tuple(), + self.useragent, + self.stealth, + ) + ) + self.context_options["extra_http_headers"] = dict( + self.context_options["extra_http_headers"] + ) + self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + + +class StealthySessionMixin: + def __validate__( + self, + max_pages, + headless, + block_images, + disable_resources, + block_webrtc, + allow_webgl, + network_idle, + humanize, + solve_cloudflare, + wait, + timeout, + page_action, + wait_selector, + init_script, + addons, + wait_selector_state, + cookies, + google_search, + extra_headers, + proxy, + os_randomize, + disable_ads, + geoip, + selector_config, + additional_args, + ): + params = { + "max_pages": max_pages, + "headless": headless, + "block_images": block_images, + "disable_resources": disable_resources, + "block_webrtc": block_webrtc, + "allow_webgl": allow_webgl, + "network_idle": network_idle, + "humanize": humanize, + "solve_cloudflare": solve_cloudflare, + "wait": wait, + "timeout": timeout, + "page_action": page_action, + "wait_selector": wait_selector, + "init_script": init_script, + "addons": addons, + "wait_selector_state": wait_selector_state, + "cookies": cookies, + "google_search": google_search, + "extra_headers": extra_headers, + "proxy": proxy, + "os_randomize": os_randomize, + "disable_ads": disable_ads, + "geoip": geoip, + "selector_config": selector_config, + "additional_args": additional_args, + } + config = validate(params, CamoufoxConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.block_images = config.block_images + self.disable_resources = config.disable_resources + self.block_webrtc = config.block_webrtc + self.allow_webgl = config.allow_webgl + self.network_idle = config.network_idle + self.humanize = config.humanize + self.solve_cloudflare = config.solve_cloudflare + self.wait = config.wait + self.timeout = config.timeout + self.page_action = config.page_action + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.addons = config.addons + self.wait_selector_state = config.wait_selector_state + self.cookies = config.cookies + self.google_search = config.google_search + self.extra_headers = config.extra_headers + self.proxy = config.proxy + self.os_randomize = config.os_randomize + self.disable_ads = config.disable_ads + self.geoip = config.geoip + self.selector_config = config.selector_config + self.additional_args = config.additional_args + self.selector_config = config.selector_config + self.page_action = config.page_action + self._headers_keys = ( + set(map(str.lower, self.extra_headers.keys())) + if self.extra_headers + else set() + ) + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + """Initiate browser options.""" + self.launch_options = generate_launch_options( + **{ + "geoip": self.geoip, + "proxy": dict(self.proxy) if self.proxy else self.proxy, + "addons": self.addons, + "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], + "headless": self.headless, + "humanize": True if self.solve_cloudflare else self.humanize, + "i_know_what_im_doing": True, # To turn warnings off with the user configurations + "allow_webgl": self.allow_webgl, + "block_webrtc": self.block_webrtc, + "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. + "os": None if self.os_randomize else get_os_name(), + "user_data_dir": "", + "ff_version": __ff_version_str__, + "firefox_user_prefs": { + # This is what enabling `enable_cache` does internally, so we do it from here instead + "browser.sessionhistory.max_entries": 10, + "browser.sessionhistory.max_total_viewers": -1, + "browser.cache.memory.enable": True, + "browser.cache.disk_cache_ssl": True, + "browser.cache.disk.smart_size.enabled": True, + }, + **self.additional_args, + } + ) + + @staticmethod + def _detect_cloudflare(page_content: str) -> str | None: + """ + Detect the type of Cloudflare challenge present in the provided page content. + + This function analyzes the given page content to identify whether a specific + type of Cloudflare challenge is present. It checks for three predefined + challenge types: non-interactive, managed, and interactive. If a challenge + type is detected, it returns the corresponding type as a string. If no + challenge type is detected, it returns None. + + Args: + page_content (str): The content of the page to analyze for Cloudflare + challenge types. + + Returns: + str: A string representing the detected Cloudflare challenge type, if + found. Returns None if no challenge matches. + """ + challenge_types = ( + "non-interactive", + "managed", + "interactive", + ) + for ctype in challenge_types: + if f"cType: '{ctype}'" in page_content: + return ctype + + return None diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 5afccb1..cd814c1 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -1,17 +1,8 @@ -from time import time, sleep from re import compile as re_compile -from asyncio import sleep as asyncio_sleep, Lock -from camoufox import DefaultAddons -from camoufox.utils import ( - launch_options as generate_launch_options, - installed_verstr as camoufox_version, -) from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, - Playwright, Locator, Page, ) @@ -24,9 +15,8 @@ from playwright.async_api import ( Page as async_Page, ) +from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log -from ._page import PageInfo, PagePool -from ._validators import validate, CamoufoxConfig from scrapling.core._types import ( Dict, List, @@ -37,17 +27,13 @@ from scrapling.core._types import ( from scrapling.engines.toolbelt import ( Response, ResponseFactory, - async_intercept_route, generate_convincing_referer, - get_os_name, - intercept_route, ) __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -__ff_version_str__ = camoufox_version().split(".", 1)[0] -class StealthySession: +class StealthySession(StealthySessionMixin, SyncSession): """A Stealthy session manager with page pooling.""" __slots__ = ( @@ -87,7 +73,7 @@ class StealthySession: def __init__( self, - max_pages: int = 1, + __max_pages: int = 1, headless: bool = True, # noqa: F821 block_images: bool = False, disable_resources: bool = False, @@ -141,107 +127,38 @@ class StealthySession: :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - params = { - "max_pages": max_pages, - "headless": headless, - "block_images": block_images, - "disable_resources": disable_resources, - "block_webrtc": block_webrtc, - "allow_webgl": allow_webgl, - "network_idle": network_idle, - "humanize": humanize, - "solve_cloudflare": solve_cloudflare, - "wait": wait, - "timeout": timeout, - "page_action": page_action, - "wait_selector": wait_selector, - "init_script": init_script, - "addons": addons, - "wait_selector_state": wait_selector_state, - "cookies": cookies, - "google_search": google_search, - "extra_headers": extra_headers, - "proxy": proxy, - "os_randomize": os_randomize, - "disable_ads": disable_ads, - "geoip": geoip, - "selector_config": selector_config, - "additional_args": additional_args, - } - config = validate(params, CamoufoxConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.block_images = config.block_images - self.disable_resources = config.disable_resources - self.block_webrtc = config.block_webrtc - self.allow_webgl = config.allow_webgl - self.network_idle = config.network_idle - self.humanize = config.humanize - self.solve_cloudflare = config.solve_cloudflare - self.wait = config.wait - self.timeout = config.timeout - self.page_action = config.page_action - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.addons = config.addons - self.wait_selector_state = config.wait_selector_state - self.cookies = config.cookies - self.google_search = config.google_search - self.extra_headers = config.extra_headers - self.proxy = config.proxy - self.os_randomize = config.os_randomize - self.disable_ads = config.disable_ads - self.geoip = config.geoip - self.selector_config = config.selector_config - self.additional_args = config.additional_args - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() - ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - """Initiate browser options.""" - self.launch_options = generate_launch_options( - **{ - "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy else self.proxy, - "addons": self.addons, - "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], - "headless": self.headless, - "humanize": True if self.solve_cloudflare else self.humanize, - "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self.allow_webgl, - "block_webrtc": self.block_webrtc, - "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self.os_randomize else get_os_name(), - "user_data_dir": "", - "ff_version": __ff_version_str__, - "firefox_user_prefs": { - # This is what enabling `enable_cache` does internally, so we do it from here instead - "browser.sessionhistory.max_entries": 10, - "browser.sessionhistory.max_total_viewers": -1, - "browser.cache.memory.enable": True, - "browser.cache.disk_cache_ssl": True, - "browser.cache.disk.smart_size.enabled": True, - }, - **self.additional_args, - } + self.__validate__( + __max_pages, + headless, + block_images, + disable_resources, + block_webrtc, + allow_webgl, + network_idle, + humanize, + solve_cloudflare, + wait, + timeout, + page_action, + wait_selector, + init_script, + addons, + wait_selector_state, + cookies, + google_search, + extra_headers, + proxy, + os_randomize, + disable_ads, + geoip, + selector_config, + additional_args, ) + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" @@ -284,68 +201,6 @@ class StealthySession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - @staticmethod - def _detect_cloudflare(page_content): - """ - Detect the type of Cloudflare challenge present in the provided page content. - - This function analyzes the given page content to identify whether a specific - type of Cloudflare challenge is present. It checks for three predefined - challenge types: non-interactive, managed, and interactive. If a challenge - type is detected, it returns the corresponding type as a string. If no - challenge type is detected, it returns None. - - Args: - page_content (str): The content of the page to analyze for Cloudflare - challenge types. - - Returns: - str: A string representing the detected Cloudflare challenge type, if - found. Returns None if no challenge matches. - """ - challenge_types = ( - "non-interactive", - "managed", - "interactive", - ) - for ctype in challenge_types: - if f"cType: '{ctype}'" in page_content: - return ctype - - return None - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -416,7 +271,7 @@ class StealthySession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page() page_info.mark_busy(url=url) try: # pragma: no cover @@ -462,8 +317,8 @@ class StealthySession: page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -471,17 +326,8 @@ class StealthySession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncStealthySession(StealthySession): +class AsyncStealthySession(StealthySessionMixin, AsyncSession): """A Stealthy session manager with page pooling.""" def __init__( @@ -544,7 +390,7 @@ class AsyncStealthySession(StealthySession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - super().__init__( + self.__validate__( max_pages, headless, block_images, @@ -571,11 +417,7 @@ class AsyncStealthySession(StealthySession): selector_config, additional_args, ) - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -618,39 +460,6 @@ class AsyncStealthySession(StealthySession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version @@ -723,7 +532,7 @@ class AsyncStealthySession(StealthySession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page() page_info.mark_busy(url=url) try: @@ -771,8 +580,8 @@ class AsyncStealthySession(StealthySession): page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 85db198..ef3b906 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -1,10 +1,6 @@ -from time import time, sleep -from asyncio import sleep as asyncio_sleep, Lock - from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, Playwright, Locator, ) @@ -21,9 +17,7 @@ from rebrowser_playwright.async_api import ( ) from scrapling.core.utils import log -from ._page import PageInfo, PagePool -from ._validators import validate, PlaywrightConfig -from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs +from ._base import SyncSession, AsyncSession, DynamicSessionMixin from scrapling.core._types import ( Dict, List, @@ -35,12 +29,10 @@ from scrapling.engines.toolbelt import ( Response, ResponseFactory, generate_convincing_referer, - intercept_route, - async_intercept_route, ) -class DynamicSession: +class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" __slots__ = ( @@ -127,108 +119,31 @@ class DynamicSession: :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - - params = { - "max_pages": __max_pages, - "headless": headless, - "google_search": google_search, - "hide_canvas": hide_canvas, - "disable_webgl": disable_webgl, - "real_chrome": real_chrome, - "stealth": stealth, - "wait": wait, - "page_action": page_action, - "proxy": proxy, - "locale": locale, - "extra_headers": extra_headers, - "useragent": useragent, - "timeout": timeout, - "selector_config": selector_config, - "disable_resources": disable_resources, - "wait_selector": wait_selector, - "init_script": init_script, - "cookies": cookies, - "network_idle": network_idle, - "wait_selector_state": wait_selector_state, - "cdp_url": cdp_url, - } - config = validate(params, PlaywrightConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.hide_canvas = config.hide_canvas - self.disable_webgl = config.disable_webgl - self.real_chrome = config.real_chrome - self.stealth = config.stealth - self.google_search = config.google_search - self.wait = config.wait - self.proxy = config.proxy - self.locale = config.locale - self.extra_headers = config.extra_headers - self.useragent = config.useragent - self.timeout = config.timeout - self.cookies = config.cookies - self.disable_resources = config.disable_resources - self.cdp_url = config.cdp_url - self.network_idle = config.network_idle - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.wait_selector_state = config.wait_selector_state - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() + self.__validate__( + __max_pages, + headless, + google_search, + hide_canvas, + disable_webgl, + real_chrome, + stealth, + wait, + page_action, + proxy, + locale, + extra_headers, + useragent, + cdp_url, + timeout, + disable_resources, + wait_selector, + init_script, + cookies, + network_idle, + wait_selector_state, + selector_config, ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - if not self.cdp_url: - # `launch_options` is used with persistent context - self.launch_options = dict( - _launch_kwargs( - self.headless, - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.real_chrome, - self.stealth, - self.hide_canvas, - self.disable_webgl, - ) - ) - self.launch_options["extra_http_headers"] = dict( - self.launch_options["extra_http_headers"] - ) - self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.context_options = dict() - else: - # while `context_options` is left to be used when cdp mode is enabled - self.launch_options = dict() - self.context_options = dict( - _context_kwargs( - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.stealth, - ) - ) - self.context_options["extra_http_headers"] = dict( - self.context_options["extra_http_headers"] - ) - self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" @@ -237,7 +152,7 @@ class DynamicSession: # Because rebrowser_playwright doesn't play well with real browsers sync_context = sync_playwright - self.playwright = sync_context().start() + self.playwright: Playwright = sync_context().start() if self.cdp_url: # pragma: no cover self.context = self.playwright.chromium.connect_over_cdp( @@ -280,43 +195,10 @@ class DynamicSession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. @@ -340,7 +222,7 @@ class DynamicSession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page() page_info.mark_busy(url=url) try: # pragma: no cover @@ -380,8 +262,8 @@ class DynamicSession: page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -389,17 +271,8 @@ class DynamicSession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncDynamicSession(DynamicSession): +class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" def __init__( @@ -455,7 +328,7 @@ class AsyncDynamicSession(DynamicSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - super().__init__( + self.__validate__( max_pages, headless, google_search, @@ -479,12 +352,7 @@ class AsyncDynamicSession(DynamicSession): wait_selector_state, selector_config, ) - - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -541,43 +409,6 @@ class AsyncDynamicSession(DynamicSession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - await page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 # seconds - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - async def fetch(self, url: str) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -602,7 +433,7 @@ class AsyncDynamicSession(DynamicSession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page() page_info.mark_busy(url=url) try: @@ -642,8 +473,8 @@ class AsyncDynamicSession(DynamicSession): page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index ec418d0..61dd51b 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,7 +6,9 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal["ready", "busy", "error"] # States that a page can be in +PageState = Literal[ + "finished", "ready", "busy", "error" +] # States that a page can be in @dataclass @@ -23,9 +25,9 @@ class PageInfo: self.state = "busy" self.url = url - def mark_ready(self): - """Mark the page as ready for new requests""" - self.state = "ready" + def mark_finished(self): + """Mark the page as finished for new requests""" + self.state = "finished" self.url = "" def mark_error(self): @@ -62,24 +64,16 @@ class PagePool: self.pages.append(page_info) return page_info - def get_ready_page(self) -> Optional[PageInfo]: - """Get a page that's ready for use""" - with self._lock: - for page_info in self.pages: - if page_info.state == "ready": - return page_info - return None - @property def pages_count(self) -> int: """Get the total number of pages""" return len(self.pages) @property - def ready_count(self) -> int: - """Get the number of ready pages""" + def finished_count(self) -> int: + """Get the number of finished pages""" with self._lock: - return sum(1 for p in self.pages if p.state == "ready") + return sum(1 for p in self.pages if p.state == "finished") @property def busy_count(self) -> int: @@ -91,3 +85,33 @@ class PagePool: """Remove pages in error state""" with self._lock: self.pages = [p for p in self.pages if p.state != "error"] + + def close_all_finished_pages(self): + """Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) + + async def aclose_all_finished_pages(self): + """Async version: Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + await page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index c1ce8f7..3e39ba2 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -118,7 +118,6 @@ class StealthyFetcher(BaseFetcher): with StealthySession( wait=wait, - max_pages=1, proxy=proxy, geoip=geoip, addons=addons, From 931993f90189a92b3b894ccf1681a0154a8f57fd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 8 Sep 2025 16:09:24 +0300 Subject: [PATCH 08/35] tests: changing tests accordingly --- tests/fetchers/sync/test_camoufox_session.py | 3 +- tests/fetchers/test_pages.py | 50 ++++---------------- 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/tests/fetchers/sync/test_camoufox_session.py b/tests/fetchers/sync/test_camoufox_session.py index c062708..4ea6f44 100644 --- a/tests/fetchers/sync/test_camoufox_session.py +++ b/tests/fetchers/sync/test_camoufox_session.py @@ -53,7 +53,6 @@ class TestStealthySession: """Test if the session is created correctly""" with StealthySession( - max_pages=3, headless=True, block_images=True, disable_resources=True, @@ -63,7 +62,7 @@ class TestStealthySession: cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}], ) as session: - assert session.max_pages == 3 + assert session.max_pages == 1 assert session.headless is True assert session.block_images is True assert session.disable_resources is True diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index fe85bd3..e3ba3bc 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -24,8 +24,8 @@ class TestPageInfo: assert page_info.state == "busy" assert page_info.url == "https://example.com" - page_info.mark_ready() - assert page_info.state == "ready" + page_info.mark_finished() + assert page_info.state == "finished" assert page_info.url == "" page_info.mark_error() @@ -63,7 +63,6 @@ class TestPagePool: assert pool.max_pages == 5 assert pool.pages_count == 0 - assert pool.ready_count == 0 assert pool.busy_count == 0 def test_add_page(self): @@ -97,42 +96,13 @@ class TestPagePool: page1 = pool.add_page(Mock()) page2 = pool.add_page(Mock()) - # Mark one as busy - page1.mark_busy("https://example.com") + # Mark them as finished + page1.mark_finished() + page2.mark_finished() - # Should get the ready page - ready_page = pool.get_ready_page() - assert ready_page == page2 - - def test_get_ready_page_none_available(self): - """Test getting ready page when none available""" - pool = PagePool(max_pages=2) - - # Add pages and mark all as busy - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - page1.mark_busy("https://example1.com") - page2.mark_busy("https://example2.com") - - # Should return None - ready_page = pool.get_ready_page() - assert ready_page is None - - def test_page_counts(self): - """Test page count properties""" - pool = PagePool(max_pages=3) - - # Add pages with different states - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - page3 = pool.add_page(Mock()) - - page1.mark_busy("https://example.com") - page3.mark_error() - - assert pool.pages_count == 3 - assert pool.ready_count == 1 - assert pool.busy_count == 1 + # test + pool.close_all_finished_pages() + assert pool.pages_count == 0 def test_cleanup_error_pages(self): """Test cleaning up error pages""" @@ -140,7 +110,7 @@ class TestPagePool: # Add pages page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) + _ = pool.add_page(Mock()) page3 = pool.add_page(Mock()) # Mark some as error @@ -151,4 +121,4 @@ class TestPagePool: pool.cleanup_error_pages() - assert pool.pages_count == 1 # Only page2 should remain + assert pool.pages_count == 1 # Only 2 should remain From 9838d741d0def5ca2235b3baf3b0dc4157b1c07a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 11 Sep 2025 03:48:55 +0300 Subject: [PATCH 09/35] refactor/feat(browser fetchers): make it possible to have a configuration per page in sessions - Also, no need for the `page_action` argument function to return the page again --- scrapling/engines/_browsers/_base.py | 37 +++-- scrapling/engines/_browsers/_camoufox.py | 171 ++++++++++++++++---- scrapling/engines/_browsers/_controllers.py | 149 ++++++++++++++--- 3 files changed, 291 insertions(+), 66 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 55b3310..6b91d8f 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -22,6 +22,7 @@ from ._config_tools import _compiled_stealth_scripts from ._validators import validate, PlaywrightConfig, CamoufoxConfig from ._config_tools import _launch_kwargs, _context_kwargs from scrapling.core._types import ( + Any, Dict, Optional, ) @@ -38,7 +39,12 @@ class SyncSession: self.context: Optional[BrowserContext] = None self._closed = False - def _get_page(self) -> PageInfo: # pragma: no cover + def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover """Get a new page to use""" # Close all finished pages to ensure clean state @@ -59,13 +65,12 @@ class SyncSession: ) page = self.context.new_page() - timeout = getattr(self, "timeout", 30000) page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) - if getattr(self, "extra_headers", False): - page.set_extra_http_headers(getattr(self, "extra_headers")) + if extra_headers: + page.set_extra_http_headers(extra_headers) - if getattr(self, "disable_resources", False): + if disable_resources: page.route("**/*", intercept_route) if getattr(self, "stealth", False): @@ -74,6 +79,13 @@ class SyncSession: return self.page_pool.add_page(page) + @staticmethod + def _get_with_precedence( + request_value: Any, session_value: Any, sentinel_value: object + ) -> Any: + """Get value with request-level priority over session-level""" + return request_value if request_value is not sentinel_value else session_value + def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" return { @@ -90,7 +102,12 @@ class AsyncSession(SyncSession): self.context: Optional[AsyncBrowserContext] = None self._lock = Lock() - async def _get_page(self) -> PageInfo: # pragma: no cover + async def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover """Get a new page to use""" async with self._lock: # Close all finished pages to ensure clean state @@ -111,13 +128,12 @@ class AsyncSession(SyncSession): ) page = await self.context.new_page() - timeout = getattr(self, "timeout", 30000) page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) - if getattr(self, "extra_headers", False): - await page.set_extra_http_headers(getattr(self, "extra_headers")) + if extra_headers: + await page.set_extra_http_headers(extra_headers) - if getattr(self, "disable_resources", False): + if disable_resources: await page.route("**/*", async_intercept_route) if getattr(self, "stealth", False): @@ -334,7 +350,6 @@ class StealthySessionMixin: self.geoip = config.geoip self.selector_config = config.selector_config self.additional_args = config.additional_args - self.selector_config = config.selector_config self.page_action = config.page_action self._headers_keys = ( set(map(str.lower, self.extra_headers.keys())) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index cd814c1..926191a 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -31,6 +31,7 @@ from scrapling.engines.toolbelt import ( ) __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +_UNSET = object() class StealthySession(StealthySessionMixin, SyncSession): @@ -247,19 +248,74 @@ class StealthySession(StealthySessionMixin, SyncSession): log.info("Cloudflare captcha is solved") return - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + solve_cloudflare = self._get_with_precedence( + solve_cloudflare, self.solve_cloudflare, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -271,7 +327,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page() + page_info = self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -280,41 +336,41 @@ class StealthySession(StealthySessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = self.page_action(page_info.page) + _ = page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(wait_selector) + waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use @@ -508,19 +564,74 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): log.info("Cloudflare captcha is solved") return - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + solve_cloudflare = self._get_with_precedence( + solve_cloudflare, self.solve_cloudflare, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -532,7 +643,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page() + page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: @@ -541,43 +652,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = await self.page_action(page_info.page) + _ = await page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(wait_selector) + await waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ef3b906..04a9d35 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -31,6 +31,8 @@ from scrapling.engines.toolbelt import ( generate_convincing_referer, ) +_UNSET = object() + class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" @@ -198,19 +200,66 @@ class DynamicSession(DynamicSessionMixin, SyncSession): def fetch( self, url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -222,7 +271,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page() + page_info = self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -231,35 +280,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = self.page_action(page_info.page) + _ = page_action(page_info.page) except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(wait_selector) + waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(wait) # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use @@ -409,19 +458,69 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self._closed = True - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -433,7 +532,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page() + page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: @@ -442,35 +541,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = await self.page_action(page_info.page) + _ = await page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(wait_selector) + await waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use From 05a714980deb7c3d930413996a5d065309d31e7f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 11 Sep 2025 04:13:36 +0300 Subject: [PATCH 10/35] docs: Update all docstring according to the new changes --- docs/fetching/dynamic.md | 13 ++++++++----- docs/fetching/stealthy.md | 13 ++++++++----- scrapling/engines/_browsers/_camoufox.py | 8 ++++---- scrapling/engines/_browsers/_controllers.py | 8 ++++---- scrapling/fetchers.py | 8 ++++---- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index e6566c5..29bf1d6 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -77,7 +77,7 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | | timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | -| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | +| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | | init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | @@ -134,7 +134,6 @@ def scroll_page(page: Page): page.mouse.wheel(10, 0) page.mouse.move(100, 400) page.mouse.up() - return page page = DynamicFetcher.fetch( 'https://example.com', @@ -149,7 +148,6 @@ async def scroll_page(page: Page): await page.mouse.wheel(10, 0) await page.mouse.move(100, 400) await page.mouse.up() - return page page = await DynamicFetcher.async_fetch( 'https://example.com', @@ -273,9 +271,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit of the maximum number of pages allowed and with each request, the library will close all tabs that finished its task and check if the number of the current tabs is lower than the number of maximum allowed number of pages/tabs then: -When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you and then all is as normal. +2. Otherwise, it will keep checking every sub second if creating a new tab is allowed or not for 60 seconds then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. + +This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) + +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time but this logic proved to have flaws since it's nearly impossible to protections pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 667f43e..f090aa5 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -31,7 +31,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ | | block_webrtc | Blocks WebRTC entirely. | ✔️ | -| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | +| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ | | addons | List of Firefox addons to use. **Must be paths to extracted addons.** | ✔️ | | humanize | Humanize the cursor movement. The cursor movement takes either True or the maximum duration in seconds. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ | | allow_webgl | Enabled by default. Disabling WebGL is not recommended, as many WAFs now check if WebGL is enabled. | ✔️ | @@ -156,7 +156,6 @@ def scroll_page(page: Page): page.mouse.wheel(10, 0) page.mouse.move(100, 400) page.mouse.up() - return page page = StealthyFetcher.fetch( 'https://example.com', @@ -171,7 +170,6 @@ async def scroll_page(page: Page): await page.mouse.wheel(10, 0) await page.mouse.move(100, 400) await page.mouse.up() - return page page = await StealthyFetcher.async_fetch( 'https://example.com', @@ -278,9 +276,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit of the maximum number of pages allowed and with each request, the library will close all tabs that finished its task and check if the number of the current tabs is lower than the number of maximum allowed number of pages/tabs then: -When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you and then all is as normal. +2. Otherwise, it will keep checking every sub second if creating a new tab is allowed or not for 60 seconds then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. + +This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) + +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time but this logic proved to have flaws since it's nearly impossible to protections pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 926191a..87526ef 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -119,7 +119,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -269,7 +269,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -433,7 +433,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -585,7 +585,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 04a9d35..d4f238f 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -106,7 +106,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -217,7 +217,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -360,7 +360,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -478,7 +478,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 3e39ba2..ea7f749 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -96,7 +96,7 @@ class StealthyFetcher(BaseFetcher): :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -194,7 +194,7 @@ class StealthyFetcher(BaseFetcher): :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -299,7 +299,7 @@ class DynamicFetcher(BaseFetcher): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -385,7 +385,7 @@ class DynamicFetcher(BaseFetcher): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. From e4b1e00e63e392b1142b754e3a459a8964988301 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 11 Sep 2025 04:14:32 +0300 Subject: [PATCH 11/35] build: Pump up the version --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index f08e652..d2b54ce 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.1" +__version__ = "0.3.2" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index ae28f61..1cac942 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.1 +version = 0.3.2 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From adb82008e5102627f372cd0555db6d7c5a2304e2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 12 Sep 2025 04:33:00 +0300 Subject: [PATCH 12/35] style: Using keywords for validation Less lines of code and more stylish --- scrapling/engines/_browsers/_base.py | 110 +------------------- scrapling/engines/_browsers/_camoufox.py | 100 +++++++++--------- scrapling/engines/_browsers/_controllers.py | 88 ++++++++-------- 3 files changed, 98 insertions(+), 200 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6b91d8f..e27591f 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -144,56 +144,8 @@ class AsyncSession(SyncSession): class DynamicSessionMixin: - def __validate__( - self, - __max_pages, - headless, - google_search, - hide_canvas, - disable_webgl, - real_chrome, - stealth, - wait, - page_action, - proxy, - locale, - extra_headers, - useragent, - cdp_url, - timeout, - disable_resources, - wait_selector, - init_script, - cookies, - network_idle, - wait_selector_state, - selector_config, - ): - params = { - "max_pages": __max_pages, - "headless": headless, - "google_search": google_search, - "hide_canvas": hide_canvas, - "disable_webgl": disable_webgl, - "real_chrome": real_chrome, - "stealth": stealth, - "wait": wait, - "page_action": page_action, - "proxy": proxy, - "locale": locale, - "extra_headers": extra_headers, - "useragent": useragent, - "timeout": timeout, - "selector_config": selector_config, - "disable_resources": disable_resources, - "wait_selector": wait_selector, - "init_script": init_script, - "cookies": cookies, - "network_idle": network_idle, - "wait_selector_state": wait_selector_state, - "cdp_url": cdp_url, - } - config = validate(params, PlaywrightConfig) + def __validate__(self, **params): + config = validate(**params, model=PlaywrightConfig) self.max_pages = config.max_pages self.headless = config.headless @@ -268,62 +220,8 @@ class DynamicSessionMixin: class StealthySessionMixin: - def __validate__( - self, - max_pages, - headless, - block_images, - disable_resources, - block_webrtc, - allow_webgl, - network_idle, - humanize, - solve_cloudflare, - wait, - timeout, - page_action, - wait_selector, - init_script, - addons, - wait_selector_state, - cookies, - google_search, - extra_headers, - proxy, - os_randomize, - disable_ads, - geoip, - selector_config, - additional_args, - ): - params = { - "max_pages": max_pages, - "headless": headless, - "block_images": block_images, - "disable_resources": disable_resources, - "block_webrtc": block_webrtc, - "allow_webgl": allow_webgl, - "network_idle": network_idle, - "humanize": humanize, - "solve_cloudflare": solve_cloudflare, - "wait": wait, - "timeout": timeout, - "page_action": page_action, - "wait_selector": wait_selector, - "init_script": init_script, - "addons": addons, - "wait_selector_state": wait_selector_state, - "cookies": cookies, - "google_search": google_search, - "extra_headers": extra_headers, - "proxy": proxy, - "os_randomize": os_randomize, - "disable_ads": disable_ads, - "geoip": geoip, - "selector_config": selector_config, - "additional_args": additional_args, - } - config = validate(params, CamoufoxConfig) + def __validate__(self, **params): + config = validate(**params, model=CamoufoxConfig) self.max_pages = config.max_pages self.headless = config.headless diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 87526ef..3412fd6 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -133,31 +133,31 @@ class StealthySession(StealthySessionMixin, SyncSession): """ self.__validate__( - __max_pages, - headless, - block_images, - disable_resources, - block_webrtc, - allow_webgl, - network_idle, - humanize, - solve_cloudflare, - wait, - timeout, - page_action, - wait_selector, - init_script, - addons, - wait_selector_state, - cookies, - google_search, - extra_headers, - proxy, - os_randomize, - disable_ads, - geoip, - selector_config, - additional_args, + wait=wait, + proxy=proxy, + geoip=geoip, + addons=addons, + timeout=timeout, + cookies=cookies, + headless=headless, + humanize=humanize, + max_pages=__max_pages, + disable_ads=disable_ads, + allow_webgl=allow_webgl, + page_action=page_action, + init_script=init_script, + network_idle=network_idle, + block_images=block_images, + block_webrtc=block_webrtc, + os_randomize=os_randomize, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + additional_args=additional_args, + selector_config=selector_config, + solve_cloudflare=solve_cloudflare, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) super().__init__(max_pages=self.max_pages) @@ -447,31 +447,31 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__( - max_pages, - headless, - block_images, - disable_resources, - block_webrtc, - allow_webgl, - network_idle, - humanize, - solve_cloudflare, - wait, - timeout, - page_action, - wait_selector, - init_script, - addons, - wait_selector_state, - cookies, - google_search, - extra_headers, - proxy, - os_randomize, - disable_ads, - geoip, - selector_config, - additional_args, + wait=wait, + proxy=proxy, + geoip=geoip, + addons=addons, + timeout=timeout, + cookies=cookies, + headless=headless, + humanize=humanize, + max_pages=max_pages, + disable_ads=disable_ads, + allow_webgl=allow_webgl, + page_action=page_action, + init_script=init_script, + network_idle=network_idle, + block_images=block_images, + block_webrtc=block_webrtc, + os_randomize=os_randomize, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + additional_args=additional_args, + selector_config=selector_config, + solve_cloudflare=solve_cloudflare, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) super().__init__(max_pages=self.max_pages) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index d4f238f..49f6c9d 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -122,28 +122,28 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ self.__validate__( - __max_pages, - headless, - google_search, - hide_canvas, - disable_webgl, - real_chrome, - stealth, - wait, - page_action, - proxy, - locale, - extra_headers, - useragent, - cdp_url, - timeout, - disable_resources, - wait_selector, - init_script, - cookies, - network_idle, - wait_selector_state, - selector_config, + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + stealth=stealth, + cdp_url=cdp_url, + cookies=cookies, + headless=headless, + useragent=useragent, + max_pages=__max_pages, + real_chrome=real_chrome, + page_action=page_action, + hide_canvas=hide_canvas, + init_script=init_script, + network_idle=network_idle, + google_search=google_search, + extra_headers=extra_headers, + wait_selector=wait_selector, + disable_webgl=disable_webgl, + selector_config=selector_config, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) super().__init__(max_pages=self.max_pages) @@ -378,28 +378,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """ self.__validate__( - max_pages, - headless, - google_search, - hide_canvas, - disable_webgl, - real_chrome, - stealth, - wait, - page_action, - proxy, - locale, - extra_headers, - useragent, - cdp_url, - timeout, - disable_resources, - wait_selector, - init_script, - cookies, - network_idle, - wait_selector_state, - selector_config, + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + stealth=stealth, + cdp_url=cdp_url, + cookies=cookies, + headless=headless, + useragent=useragent, + max_pages=max_pages, + real_chrome=real_chrome, + page_action=page_action, + hide_canvas=hide_canvas, + init_script=init_script, + network_idle=network_idle, + google_search=google_search, + extra_headers=extra_headers, + wait_selector=wait_selector, + disable_webgl=disable_webgl, + selector_config=selector_config, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) super().__init__(max_pages=self.max_pages) From 0c7db3c9d90f006c4920457811faa1969f53f2fb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 02:07:04 +0300 Subject: [PATCH 13/35] fix: validation keywords --- scrapling/engines/_browsers/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index e27591f..23a77d6 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -145,7 +145,7 @@ class AsyncSession(SyncSession): class DynamicSessionMixin: def __validate__(self, **params): - config = validate(**params, model=PlaywrightConfig) + config = validate(params, model=PlaywrightConfig) self.max_pages = config.max_pages self.headless = config.headless @@ -221,7 +221,7 @@ class DynamicSessionMixin: class StealthySessionMixin: def __validate__(self, **params): - config = validate(**params, model=CamoufoxConfig) + config = validate(params, model=CamoufoxConfig) self.max_pages = config.max_pages self.headless = config.headless From 831eafca34328f4ddf66f388058fdc4969f7102b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:07:15 +0300 Subject: [PATCH 14/35] ops: Update ruff pre-commit rules --- ruff.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ruff.toml b/ruff.toml index a579697..405614a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -10,8 +10,10 @@ exclude = [ "benchmarks.py", ] -# Assume Python 3.9 -target-version = "py39" +# Assume Python 3.10 +target-version = "py310" +# Allow lines to be as long as 120. +line-length = 120 [lint] select = ["E", "F", "W"] From 60be9dc816b5bf82adf83534c712c28156f9a17f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:22:08 +0300 Subject: [PATCH 15/35] feat: Validate all fetch-level parameters So it's validated like the session-level ones. Needs improvement, but that's for later --- scrapling/engines/_browsers/_camoufox.py | 160 ++++++++------------ scrapling/engines/_browsers/_controllers.py | 153 ++++++++----------- 2 files changed, 128 insertions(+), 185 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 3412fd6..fcb77ac 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -15,6 +15,7 @@ from playwright.async_api import ( Page as async_Page, ) +from ._validators import validate, CamoufoxConfig from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( @@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession): def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() - self.context = ( - self.playwright.firefox.launch_persistent_context( # pragma: no cover - **self.launch_options - ) + self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover + **self.launch_options ) # Get the default page and close it @@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - solve_cloudflare = self._get_with_precedence( - solve_cloudflare, self.solve_cloudflare, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, ) if self._closed: # pragma: no cover @@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page(timeout, extra_headers, disable_resources) + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if solve_cloudflare: + if params.solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") - if page_action is not None: + if params.page_action: try: - _ = page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(wait_selector) - waiter.first.wait_for(state=wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(wait) + page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use @@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): async def __create__(self): """Create a browser for this instance and context.""" self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = ( - await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( + **self.launch_options ) # Get the default page and close it @@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await page.wait_for_timeout(500) # Calculate the Captcha coordinates for any viewport - outer_box = await page.locator( - ".main-content p+div>div>div" - ).bounding_box() + outer_box = await page.locator(".main-content p+div>div>div").bounding_box() captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 # Move the mouse to the center of the window, then press and hold the left mouse button @@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - solve_cloudflare = self._get_with_precedence( - solve_cloudflare, self.solve_cloudflare, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, ) if self._closed: # pragma: no cover @@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page(timeout, extra_headers, disable_resources) + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: @@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if solve_cloudflare: + if params.solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") - if page_action is not None: + if params.page_action: try: - _ = await page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(wait_selector) - await waiter.first.wait_for(state=wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 49f6c9d..a53fa62 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import ( from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin +from ._validators import validate, PlaywrightConfig from scrapling.core._types import ( Dict, List, @@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession): self.playwright: Playwright = sync_context().start() if self.cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ).new_context(**self.context_options) - else: - self.context = self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( + **self.context_options ) + else: + self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options) # Get the default page and close it default_page = self.context.pages[0] @@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, ) if self._closed: # pragma: no cover @@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page(timeout, extra_headers, disable_resources) + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if page_action is not None: + if params.page_action: try: - _ = page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(wait_selector) - waiter.first.wait_for(state=wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(wait) + page_info.page.wait_for_timeout(params.wait) # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use @@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self.playwright: AsyncPlaywright = await async_context().start() if self.cdp_url: - browser = await self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ) - self.context: AsyncBrowserContext = await browser.new_context( - **self.context_options - ) + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: - self.context: AsyncBrowserContext = ( - await self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + user_data_dir="", **self.launch_options ) # Get the default page and close it @@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, ) if self._closed: # pragma: no cover @@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page(timeout, extra_headers, disable_resources) + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: @@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if page_action is not None: + if params.page_action: try: - _ = await page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(wait_selector) - await waiter.first.wait_for(state=wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use From 330d03559cf70624ba952d346f991c72da6b069a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:22:53 +0300 Subject: [PATCH 16/35] style: applying the new ruff rules to all files --- scrapling/cli.py | 61 +++------ scrapling/core/_html_utils.py | 12 +- scrapling/core/ai.py | 16 +-- scrapling/core/custom_types.py | 80 +++--------- scrapling/core/mixins.py | 34 +---- scrapling/core/shell.py | 82 +++--------- scrapling/core/storage.py | 10 +- scrapling/core/translator.py | 24 +--- scrapling/core/utils.py | 26 +--- scrapling/engines/_browsers/_base.py | 32 +---- scrapling/engines/_browsers/_page.py | 4 +- scrapling/engines/_browsers/_validators.py | 20 +-- scrapling/engines/static.py | 90 ++++--------- scrapling/engines/toolbelt/convertor.py | 52 ++------ scrapling/engines/toolbelt/custom.py | 28 +--- scrapling/engines/toolbelt/navigation.py | 17 +-- scrapling/fetchers.py | 16 +-- scrapling/parser.py | 144 +++++---------------- 18 files changed, 176 insertions(+), 572 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index a914ffe..e4b3c7f 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -72,14 +72,10 @@ def __ParseExtractArguments( return parsed_headers, parsed_cookies, parsed_params, parsed_json -def __BuildRequest( - headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs -) -> Dict: +def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs) -> Dict: """Build a request object using the specified arguments""" # Parse parameters - parsed_headers, parsed_cookies, parsed_params, parsed_json = ( - __ParseExtractArguments(headers, cookies, params, json) - ) + parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json) # Build request arguments request_kwargs = { "headers": parsed_headers if parsed_headers else None, @@ -106,10 +102,7 @@ def __BuildRequest( help="Force Scrapling to reinstall all Fetchers dependencies", ) def install(force): # pragma: no cover - if ( - force - or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists() - ): + if force or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists(): __Execute( [python_executable, "-m", "playwright", "install", "chromium"], "Playwright browsers", @@ -158,9 +151,7 @@ def mcp(): "level", is_flag=False, default="debug", - type=Choice( - ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False - ), + type=Choice(["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False), help="Log level (default: DEBUG)", ) def shell(code, level): @@ -178,9 +169,7 @@ def extract(): pass -@extract.command( - help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -190,9 +179,7 @@ def extract(): help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -267,9 +254,7 @@ def get( __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -285,9 +270,7 @@ def get( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -367,9 +350,7 @@ def post( __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option("--data", "-d", help="Form data to include in the request body") @@ -381,9 +362,7 @@ def post( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -463,9 +442,7 @@ def put( __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -475,9 +452,7 @@ def put( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -552,9 +527,7 @@ def delete( __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -591,9 +564,7 @@ def delete( ) @option("--wait-selector", help="CSS selector to wait for before proceeding") @option("--locale", default="en-US", help="Browser locale (default: en-US)") -@option( - "--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)" -) +@option("--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)") @option( "--hide-canvas/--show-canvas", default=False, @@ -675,9 +646,7 @@ def fetch( __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( diff --git a/scrapling/core/_html_utils.py b/scrapling/core/_html_utils.py index 99776af..6b09830 100644 --- a/scrapling/core/_html_utils.py +++ b/scrapling/core/_html_utils.py @@ -269,17 +269,13 @@ name2codepoint = { } -def to_unicode( - text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict" -) -> str: +def to_unicode(text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict") -> str: """Return the Unicode representation of a bytes object `text`. If `text` is already a Unicode object, return it as-is.""" if isinstance(text, str): return text if not isinstance(text, (bytes, str)): - raise TypeError( - f"to_unicode must receive bytes or str, got {type(text).__name__}" - ) + raise TypeError(f"to_unicode must receive bytes or str, got {type(text).__name__}") if encoding is None: encoding = "utf-8" return text.decode(encoding, errors) @@ -328,9 +324,7 @@ def _replace_entities( entity_name = groups["named"] if entity_name.lower() in keep: return m.group(0) - number = name2codepoint.get(entity_name) or name2codepoint.get( - entity_name.lower() - ) + number = name2codepoint.get(entity_name) or name2codepoint.get(entity_name.lower()) if number is not None: # Browsers typically # interpret numeric character references in the 80-9F range as representing the characters mapped diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index aa2d52b..7777c21 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -32,21 +32,13 @@ class ResponseModel(BaseModel): """Request's response information structure.""" status: int = Field(description="The status code returned by the website.") - content: list[str] = Field( - description="The content as Markdown/HTML or the text content of the page." - ) - url: str = Field( - description="The URL given by the user that resulted in this response." - ) + content: list[str] = Field(description="The content as Markdown/HTML or the text content of the page.") + url: str = Field(description="The URL given by the user that resulted in this response.") -def _ContentTranslator( - content: Generator[str, None, None], page: _ScraplingResponse -) -> ResponseModel: +def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: """Convert a content generator to a list of ResponseModel objects.""" - return ResponseModel( - status=page.status, content=[result for result in content], url=page.url - ) + return ResponseModel(status=page.status, content=[result for result in content], url=page.url) class ScraplingMCPServer: diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index ef76880..eb7fa34 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -31,15 +31,11 @@ class TextHandler(str): __slots__ = () - def __getitem__( - self, key: SupportsIndex | slice - ) -> "TextHandler": # pragma: no cover + def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": # pragma: no cover lst = super().__getitem__(key) return cast(_TextHandlerType, TextHandler(lst)) - def split( - self, sep: str = None, maxsplit: SupportsIndex = -1 - ) -> "TextHandlers": # pragma: no cover + def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": # pragma: no cover return TextHandlers( cast( List[_TextHandlerType], @@ -50,14 +46,10 @@ class TextHandler(str): def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().strip(chars)) - def lstrip( - self, chars: str = None - ) -> Union[str, "TextHandler"]: # pragma: no cover + def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().lstrip(chars)) - def rstrip( - self, chars: str = None - ) -> Union[str, "TextHandler"]: # pragma: no cover + def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().rstrip(chars)) def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover @@ -66,37 +58,25 @@ class TextHandler(str): def casefold(self) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().casefold()) - def center( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def center(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().center(width, fillchar)) - def expandtabs( - self, tabsize: SupportsIndex = 8 - ) -> Union[str, "TextHandler"]: # pragma: no cover + def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().expandtabs(tabsize)) - def format( - self, *args: str, **kwargs: str - ) -> Union[str, "TextHandler"]: # pragma: no cover + def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format(*args, **kwargs)) def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format_map(mapping)) - def join( - self, iterable: Iterable[str] - ) -> Union[str, "TextHandler"]: # pragma: no cover + def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().join(iterable)) - def ljust( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def ljust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().ljust(width, fillchar)) - def rjust( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def rjust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().rjust(width, fillchar)) def swapcase(self) -> Union[str, "TextHandler"]: # pragma: no cover @@ -108,14 +88,10 @@ class TextHandler(str): def translate(self, table) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().translate(table)) - def zfill( - self, width: SupportsIndex - ) -> Union[str, "TextHandler"]: # pragma: no cover + def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().zfill(width)) - def replace( - self, old: str, new: str, count: SupportsIndex = -1 - ) -> Union[str, "TextHandler"]: + def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, "TextHandler"]: return TextHandler(super().replace(old, new, count)) def upper(self) -> Union[str, "TextHandler"]: @@ -203,11 +179,7 @@ class TextHandler(str): results = flatten(results) if not replace_entities: - return TextHandlers( - cast( - List[_TextHandlerType], [TextHandler(string) for string in results] - ) - ) + return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results])) return TextHandlers( cast( @@ -257,9 +229,7 @@ class TextHandlers(List[TextHandler]): def __getitem__(self, pos: slice) -> "TextHandlers": # pragma: no cover pass - def __getitem__( - self, pos: SupportsIndex | slice - ) -> Union[TextHandler, "TextHandlers"]: + def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]: lst = super().__getitem__(pos) if isinstance(pos, slice): return TextHandlers(cast(List[_TextHandlerType], lst)) @@ -280,9 +250,7 @@ class TextHandlers(List[TextHandler]): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it """ - results = [ - n.re(regex, replace_entities, clean_match, case_sensitive) for n in self - ] + results = [n.re(regex, replace_entities, clean_match, case_sensitive) for n in self] return TextHandlers(flatten(results)) def re_first( @@ -330,34 +298,24 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): def __init__(self, mapping=None, **kwargs): mapping = ( - { - key: TextHandler(value) if isinstance(value, str) else value - for key, value in mapping.items() - } + {key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()} if mapping is not None else {} ) if kwargs: mapping.update( - { - key: TextHandler(value) if isinstance(value, str) else value - for key, value in kwargs.items() - } + {key: TextHandler(value) if isinstance(value, str) else value for key, value in kwargs.items()} ) # Fastest read-only mapping type self._data = MappingProxyType(mapping) - def get( - self, key: str, default: Optional[str] = None - ) -> Optional[_TextHandlerType]: + def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]: """Acts like the standard dictionary `.get()` method""" return self._data.get(key, default) - def search_values( - self, keyword: str, partial: bool = False - ) -> Generator["AttributesHandler", None, None]: + def search_values(self, keyword: str, partial: bool = False) -> Generator["AttributesHandler", None, None]: """Search current attributes by values and return a dictionary of each matching item :param keyword: The keyword to search for in the attribute values :param partial: If True, the function will search if keyword in each value instead of perfect match diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py index afad094..4087020 100644 --- a/scrapling/core/mixins.py +++ b/scrapling/core/mixins.py @@ -5,9 +5,7 @@ class SelectorsGeneration: Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591 """ - def __general_selection( - self, selection: str = "css", full_path: bool = False - ) -> str: + def __general_selection(self, selection: str = "css", full_path: bool = False) -> str: """Generate a selector for the current element. :return: A string of the generated selector. """ @@ -18,18 +16,10 @@ class SelectorsGeneration: if target.parent: if target.attrib.get("id"): # id is enough - part = ( - f"#{target.attrib['id']}" - if css - else f"[@id='{target.attrib['id']}']" - ) + part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']" selectorPath.append(part) if not full_path: - return ( - " > ".join(reversed(selectorPath)) - if css - else "//*" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath)) else: part = f"{target.tag}" # We won't use classes anymore because I some websites share exact classes between elements @@ -45,28 +35,16 @@ class SelectorsGeneration: break if counter[target.tag] > 1: - part += ( - f":nth-of-type({counter[target.tag]})" - if css - else f"[{counter[target.tag]}]" - ) + part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]" selectorPath.append(part) target = target.parent if target is None or target.tag == "html": - return ( - " > ".join(reversed(selectorPath)) - if css - else "//" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) else: break - return ( - " > ".join(reversed(selectorPath)) - if css - else "//" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) @property def generate_css_selector(self) -> str: diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 24e504c..8459a60 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -79,9 +79,7 @@ def _CookieParser(cookie_string): yield key, morsel.value -def _ParseHeaders( - header_lines: List[str], parse_cookies: bool = True -) -> Tuple[Dict[str, str], Dict[str, str]]: +def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: """Parses headers into separate header and cookie dictionaries.""" header_dict = dict() cookie_dict = dict() @@ -93,9 +91,7 @@ def _ParseHeaders( header_value = "" header_dict[header_key] = header_value else: - raise ValueError( - f"Could not parse header without colon: '{header_line}'." - ) + raise ValueError(f"Could not parse header without colon: '{header_line}'.") else: header_key, header_value = header_line.split(":", 1) header_key = header_key.strip() @@ -104,13 +100,9 @@ def _ParseHeaders( if parse_cookies: if header_key.lower() == "cookie": try: - cookie_dict = { - key: value for key, value in _CookieParser(header_value) - } + cookie_dict = {key: value for key, value in _CookieParser(header_value)} except Exception as e: # pragma: no cover - raise ValueError( - f"Could not parse cookie string from header '{header_value}': {e}" - ) + raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") else: header_dict[header_key] = header_value else: @@ -129,9 +121,7 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no cover if message: log.error(f"Scrapling shell exited with status {status}: {message}") self._print_message(message, stderr) - raise ValueError( - f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}" - ) + raise ValueError(f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}") class CurlParser: @@ -152,15 +142,11 @@ class CurlParser: # Data arguments (prioritizing types common from DevTools) _parser.add_argument("-d", "--data", default=None) - _parser.add_argument( - "--data-raw", default=None - ) # Often used by browsers for JSON body + _parser.add_argument("--data-raw", default=None) # Often used by browsers for JSON body _parser.add_argument("--data-binary", default=None) # Keep urlencode for completeness, though less common from browser copy/paste _parser.add_argument("--data-urlencode", action="append", default=[]) - _parser.add_argument( - "-G", "--get", action="store_true" - ) # Use GET and put data in URL + _parser.add_argument("-G", "--get", action="store_true") # Use GET and put data in URL _parser.add_argument( "-b", @@ -175,9 +161,7 @@ class CurlParser: # Connection/Security _parser.add_argument("-k", "--insecure", action="store_true") - _parser.add_argument( - "--compressed", action="store_true" - ) # Very common from browsers + _parser.add_argument("--compressed", action="store_true") # Very common from browsers # Other flags often included but may not map directly to request args _parser.add_argument("-i", "--include", action="store_true") @@ -194,9 +178,7 @@ class CurlParser: clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ") try: - tokens = shlex_split( - clean_command - ) # Split the string using shell-like syntax + tokens = shlex_split(clean_command) # Split the string using shell-like syntax except ValueError as e: # pragma: no cover log.error(f"Could not split command line: {e}") return None @@ -213,9 +195,7 @@ class CurlParser: raise except Exception as e: # pragma: no cover - log.error( - f"An unexpected error occurred during curl arguments parsing: {e}" - ) + log.error(f"An unexpected error occurred during curl arguments parsing: {e}") return None # --- Determine Method --- @@ -247,9 +227,7 @@ class CurlParser: cookies[key] = value log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}") except Exception as e: # pragma: no cover - log.error( - f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}" - ) + log.error(f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}") # --- Process Data Payload --- params = dict() @@ -280,9 +258,7 @@ class CurlParser: try: data_payload = dict(parse_qsl(combined_data, keep_blank_values=True)) except Exception as e: - log.warning( - f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string." - ) + log.warning(f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string.") data_payload = combined_data # Check if raw data looks like JSON, prefer 'json' param if so @@ -303,9 +279,7 @@ class CurlParser: try: params.update(dict(parse_qsl(data_payload, keep_blank_values=True))) except ValueError: - log.warning( - f"Could not parse data '{data_payload}' into GET parameters for -G." - ) + log.warning(f"Could not parse data '{data_payload}' into GET parameters for -G.") if params: data_payload = None # Clear data as it's moved to params @@ -314,21 +288,13 @@ class CurlParser: # --- Process Proxy --- proxies: Optional[Dict[str, str]] = None if parsed_args.proxy: - proxy_url = ( - f"http://{parsed_args.proxy}" - if "://" not in parsed_args.proxy - else parsed_args.proxy - ) + proxy_url = f"http://{parsed_args.proxy}" if "://" not in parsed_args.proxy else parsed_args.proxy if parsed_args.proxy_user: user_pass = parsed_args.proxy_user parts = urlparse(proxy_url) netloc_parts = parts.netloc.split("@") - netloc = ( - f"{user_pass}@{netloc_parts[-1]}" - if len(netloc_parts) > 1 - else f"{user_pass}@{parts.netloc}" - ) + netloc = f"{user_pass}@{netloc_parts[-1]}" if len(netloc_parts) > 1 else f"{user_pass}@{parts.netloc}" proxy_url = urlunparse( ( parts.scheme, @@ -359,11 +325,7 @@ class CurlParser: def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]: if isinstance(curl_command, (Request, str)): - request = ( - self.parse(curl_command) - if isinstance(curl_command, str) - else curl_command - ) + request = self.parse(curl_command) if isinstance(curl_command, str) else curl_command # Ensure request parsing was successful before proceeding if request is None: # pragma: no cover @@ -386,9 +348,7 @@ class CurlParser: log.error(f"Error calling Fetcher.{method}: {e}") return None else: # pragma: no cover - log.error( - f'Request method "{method}" isn\'t supported by Scrapling yet' - ) + log.error(f'Request method "{method}" isn\'t supported by Scrapling yet') return None else: # pragma: no cover @@ -621,18 +581,14 @@ class Convertor: yield "" @classmethod - def write_content_to_file( - cls, page: Selector, filename: str, css_selector: Optional[str] = None - ) -> None: + def write_content_to_file(cls, page: Selector, filename: str, css_selector: Optional[str] = None) -> None: """Write a Selector's content to a file""" if not page or not isinstance(page, Selector): # pragma: no cover raise TypeError("Input must be of type `Selector`") elif not filename or not isinstance(filename, str) or not filename.strip(): raise ValueError("Filename must be provided") elif not filename.endswith((".md", ".html", ".txt")): - raise ValueError( - "Unknown file type: filename must end with '.md', '.html', or '.txt'" - ) + raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'") else: with open(filename, "w", encoding="utf-8") as f: extension = filename.split(".")[-1] diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index 089a9ec..03c05a2 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -27,11 +27,7 @@ class StorageSystemMixin(ABC): # pragma: no cover try: extracted = tld(self.url) - return ( - extracted.top_domain_under_public_suffix - or extracted.domain - or default_value - ) + return extracted.top_domain_under_public_suffix or extracted.domain or default_value except AttributeError: return default_value @@ -90,9 +86,7 @@ class SQLiteStorageSystem(StorageSystemMixin): self.connection.execute("PRAGMA journal_mode=WAL") self.cursor = self.connection.cursor() self._setup_database() - log.debug( - f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")' - ) + log.debug(f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")') def _setup_database(self) -> None: self.cursor.execute(""" diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index e0a91bc..88dfbff 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -89,9 +89,7 @@ class TranslatorMixin: xpath = super().xpath_element(selector) # type: ignore[safe-super] return XPathExpr.from_xpath(xpath) - def xpath_pseudo_element( - self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement - ) -> OriginalXPathExpr: + def xpath_pseudo_element(self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement) -> OriginalXPathExpr: """ Dispatch method that transforms XPath to support the pseudo-element. """ @@ -99,31 +97,21 @@ class TranslatorMixin: method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element" method = getattr(self, method_name, None) if not method: # pragma: no cover - raise ExpressionError( - f"The functional pseudo-element ::{pseudo_element.name}() is unknown" - ) + raise ExpressionError(f"The functional pseudo-element ::{pseudo_element.name}() is unknown") xpath = method(xpath, pseudo_element) else: - method_name = ( - f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element" - ) + method_name = f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element" method = getattr(self, method_name, None) if not method: # pragma: no cover - raise ExpressionError( - f"The pseudo-element ::{pseudo_element} is unknown" - ) + raise ExpressionError(f"The pseudo-element ::{pseudo_element} is unknown") xpath = method(xpath) return xpath @staticmethod - def xpath_attr_functional_pseudo_element( - xpath: OriginalXPathExpr, function: FunctionalPseudoElement - ) -> XPathExpr: + def xpath_attr_functional_pseudo_element(xpath: OriginalXPathExpr, function: FunctionalPseudoElement) -> XPathExpr: """Support selecting attribute values using ::attr() pseudo-element""" if function.argument_types() not in (["STRING"], ["IDENT"]): # pragma: no cover - raise ExpressionError( - f"Expected a single string or ident for ::attr(), got {function.arguments!r}" - ) + raise ExpressionError(f"Expected a single string or ident for ::attr(), got {function.arguments!r}") return XPathExpr.from_xpath(xpath, attribute=function.arguments[0].value) @staticmethod diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index 5607f8d..2f57cfa 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -24,9 +24,7 @@ def setup_logger(): logger = logging.getLogger("scrapling") logger.setLevel(logging.INFO) - formatter = logging.Formatter( - fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S" - ) + formatter = logging.Formatter(fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) @@ -61,11 +59,7 @@ class _StorageTools: def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict: if not element.attrib: return {} - return { - k: v.strip() - for k, v in element.attrib.items() - if v and v.strip() and k not in forbidden - } + return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden} @classmethod def element_to_dict(cls, element: html.HtmlElement) -> Dict: @@ -85,17 +79,11 @@ class _StorageTools: } ) - siblings = [ - child.tag for child in parent.iterchildren() if child != element - ] + siblings = [child.tag for child in parent.iterchildren() if child != element] if siblings: result.update({"siblings": tuple(siblings)}) - children = [ - child.tag - for child in element.iterchildren() - if not isinstance(child, html_forbidden) - ] + children = [child.tag for child in element.iterchildren() if not isinstance(child, html_forbidden)] if children: result.update({"children": tuple(children)}) @@ -104,11 +92,7 @@ class _StorageTools: @classmethod def _get_element_path(cls, element: html.HtmlElement): parent = element.getparent() - return tuple( - (element.tag,) - if parent is None - else (cls._get_element_path(parent) + (element.tag,)) - ) + return tuple((element.tag,) if parent is None else (cls._get_element_path(parent) + (element.tag,))) @lru_cache(128, typed=True) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 23a77d6..6412698 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -80,9 +80,7 @@ class SyncSession: return self.page_pool.add_page(page) @staticmethod - def _get_with_precedence( - request_value: Any, session_value: Any, sentinel_value: object - ) -> Any: + def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any: """Get value with request-level priority over session-level""" return request_value if request_value is not sentinel_value else session_value @@ -169,11 +167,7 @@ class DynamicSessionMixin: self.wait_selector_state = config.wait_selector_state self.selector_config = config.selector_config self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() - ) + self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): @@ -184,9 +178,7 @@ class DynamicSessionMixin: self.headless, self.proxy, self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), + tuple(self.extra_headers.items()) if self.extra_headers else tuple(), self.useragent, self.real_chrome, self.stealth, @@ -194,9 +186,7 @@ class DynamicSessionMixin: self.disable_webgl, ) ) - self.launch_options["extra_http_headers"] = dict( - self.launch_options["extra_http_headers"] - ) + self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None self.context_options = dict() else: @@ -206,16 +196,12 @@ class DynamicSessionMixin: _context_kwargs( self.proxy, self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), + tuple(self.extra_headers.items()) if self.extra_headers else tuple(), self.useragent, self.stealth, ) ) - self.context_options["extra_http_headers"] = dict( - self.context_options["extra_http_headers"] - ) + self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None @@ -249,11 +235,7 @@ class StealthySessionMixin: self.selector_config = config.selector_config self.additional_args = config.additional_args self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() - ) + self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 61dd51b..ffa4b22 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,9 +6,7 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal[ - "finished", "ready", "busy", "error" -] # States that a page can be in +PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in @dataclass diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 6363df7..c34d6fe 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -25,9 +25,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): stealth: bool = False wait: int | float = 0 page_action: Optional[Callable] = None - proxy: Optional[str | Dict[str, str]] = ( - None # The default value for proxy in Playwright's source is `None` - ) + proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` locale: str = "en-US" extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None @@ -46,10 +44,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): raise ValueError("max_pages must be between 1 and 50") if self.timeout < 0: raise ValueError("timeout must be >= 0") - if self.page_action is not None and not callable(self.page_action): - raise TypeError( - f"page_action must be callable, got {type(self.page_action).__name__}" - ) + if self.page_action and not callable(self.page_action): + raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) if self.cdp_url: @@ -108,9 +104,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): cookies: Optional[List[Dict]] = None google_search: bool = True extra_headers: Optional[Dict[str, str]] = None - proxy: Optional[str | Dict[str, str]] = ( - None # The default value for proxy in Playwright's source is `None` - ) + proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` os_randomize: bool = False disable_ads: bool = False geoip: bool = False @@ -123,10 +117,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): raise ValueError("max_pages must be between 1 and 50") if self.timeout < 0: raise ValueError("timeout must be >= 0") - if self.page_action is not None and not callable(self.page_action): - raise TypeError( - f"page_action must be callable, got {type(self.page_action).__name__}" - ) + if self.page_action and not callable(self.page_action): + raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 59f68fa..e028feb 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -108,13 +108,9 @@ class FetcherSession: headers = self.get_with_precedence(kwargs, "headers", self.default_headers) stealth = self.get_with_precedence(kwargs, "stealth", self.stealth) - impersonate = self.get_with_precedence( - kwargs, "impersonate", self.default_impersonate - ) + impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate) - if self.get_with_precedence( - kwargs, "http3", self.default_http3 - ): # pragma: no cover + if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover request_args["http_version"] = CurlHttpVersion.V3ONLY if impersonate: log.warning( @@ -126,25 +122,13 @@ class FetcherSession: "url": url, # Curl automatically generates the suitable browser headers when you use `impersonate` "headers": self._headers_job(url, headers, stealth, bool(impersonate)), - "proxies": self.get_with_precedence( - kwargs, "proxies", self.default_proxies - ), + "proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies), "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy), - "proxy_auth": self.get_with_precedence( - kwargs, "proxy_auth", self.default_proxy_auth - ), - "timeout": self.get_with_precedence( - kwargs, "timeout", self.default_timeout - ), - "allow_redirects": self.get_with_precedence( - kwargs, "allow_redirects", self.default_follow_redirects - ), - "max_redirects": self.get_with_precedence( - kwargs, "max_redirects", self.default_max_redirects - ), - "verify": self.get_with_precedence( - kwargs, "verify", self.default_verify - ), + "proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth), + "timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout), + "allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects), + "max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects), + "verify": self.get_with_precedence(kwargs, "verify", self.default_verify), "cert": self.get_with_precedence(kwargs, "cert", self.default_cert), "impersonate": impersonate, **{ @@ -192,18 +176,12 @@ class FetcherSession: extra_headers = generate_headers(browser_mode=False) # Don't overwrite user-supplied headers - extra_headers = { - key: value - for key, value in extra_headers.items() - if key.lower() not in headers_keys - } + extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} headers.update(extra_headers) elif "user-agent" not in headers_keys and not impersonate_enabled: headers["User-Agent"] = __default_useragent__ - log.debug( - f"Can't find useragent in headers so '{headers['User-Agent']}' was used." - ) + log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") return headers @@ -215,9 +193,7 @@ class FetcherSession: "Create a new FetcherSession instance for a new independent session, " "or use the current instance sequentially after the previous context has exited." ) - if ( - self._async_curl_session - ): # Prevent mixing if async is active from this instance + if self._async_curl_session: # Prevent mixing if async is active from this instance raise RuntimeError( "This FetcherSession instance has an active asynchronous session. " "Cannot enter a synchronous context simultaneously with the same manager instance." @@ -275,9 +251,7 @@ class FetcherSession: :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ session = self._curl_session - if session is True and not any( - (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) - ): + if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): # For usage inside FetcherClient # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. session = CurlSession() @@ -290,9 +264,7 @@ class FetcherSession: return ResponseFactory.from_http_request(response, selector_config) except CurlError as e: # pragma: no cover if attempt < max_retries - 1: - log.error( - f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." - ) + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") time_sleep(retry_delay) else: log.error(f"Failed after {max_retries} attempts: {e}") @@ -320,9 +292,7 @@ class FetcherSession: :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ session = self._async_curl_session - if session is True and not any( - (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) - ): + if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. # 2. `curl_cffi` doesn't support making async requests without sessions @@ -337,9 +307,7 @@ class FetcherSession: return ResponseFactory.from_http_request(response, selector_config) except CurlError as e: # pragma: no cover if attempt < max_retries - 1: - log.error( - f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." - ) + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") await asyncio_sleep(retry_delay) else: log.error(f"Failed after {max_retries} attempts: {e}") @@ -372,19 +340,13 @@ class FetcherSession: selector_config = kwargs.pop("selector_config", {}) or self.selector_config max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) - retry_delay = self.get_with_precedence( - kwargs, "retry_delay", self.default_retry_delay - ) + retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay) request_args = self._merge_request_args(stealth=stealth, **kwargs) if self._curl_session: - return self.__make_request( - method, request_args, max_retries, retry_delay, selector_config - ) + return self.__make_request(method, request_args, max_retries, retry_delay, selector_config) elif self._async_curl_session: # The returned value is a Coroutine - return self.__make_async_request( - method, request_args, max_retries, retry_delay, selector_config - ) + return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config) raise RuntimeError("No active session available.") @@ -455,9 +417,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "GET", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args) def post( self, @@ -532,9 +492,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "POST", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args) def put( self, @@ -609,9 +567,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "PUT", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args) def delete( self, @@ -688,9 +644,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "DELETE", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args) class FetcherClient(FetcherSession): diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 12bfd55..6b0ca41 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -18,9 +18,7 @@ class ResponseFactory: """ @classmethod - def _process_response_history( - cls, first_response: SyncResponse, parser_arguments: Dict - ) -> list[Response]: + def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]: """Process response history to build a list of `Response` objects""" history = [] current_request = first_response.request.redirected_from @@ -36,18 +34,12 @@ class ResponseFactory: # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" content="", status=current_response.status if current_response else 301, - reason=( - current_response.status_text - or StatusText.get(current_response.status) - ) + reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") - or "utf-8", + encoding=current_response.headers.get("content-type", "") or "utf-8", cookies=tuple(), - headers=current_response.all_headers() - if current_response - else {}, + headers=current_response.all_headers() if current_response else {}, request_headers=current_request.all_headers(), **parser_arguments, ), @@ -94,13 +86,9 @@ class ResponseFactory: raise ValueError("Failed to get a response from the page") # This will be parsed inside `Response` - encoding = ( - final_response.headers.get("content-type", "") or "utf-8" - ) # default encoding + encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = final_response.status_text or StatusText.get( - final_response.status - ) + status_text = final_response.status_text or StatusText.get(final_response.status) history = cls._process_response_history(first_response, parser_arguments) try: @@ -141,18 +129,12 @@ class ResponseFactory: # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" content="", status=current_response.status if current_response else 301, - reason=( - current_response.status_text - or StatusText.get(current_response.status) - ) + reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") - or "utf-8", + encoding=current_response.headers.get("content-type", "") or "utf-8", cookies=tuple(), - headers=await current_response.all_headers() - if current_response - else {}, + headers=await current_response.all_headers() if current_response else {}, request_headers=await current_request.all_headers(), **parser_arguments, ), @@ -199,17 +181,11 @@ class ResponseFactory: raise ValueError("Failed to get a response from the page") # This will be parsed inside `Response` - encoding = ( - final_response.headers.get("content-type", "") or "utf-8" - ) # default encoding + encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = final_response.status_text or StatusText.get( - final_response.status - ) + status_text = final_response.status_text or StatusText.get(final_response.status) - history = await cls._async_process_response_history( - first_response, parser_arguments - ) + history = await cls._async_process_response_history(first_response, parser_arguments) try: page_content = await page.content() except Exception as e: # pragma: no cover @@ -239,9 +215,7 @@ class ResponseFactory: """ return Response( url=response.url, - content=response.content - if isinstance(response.content, bytes) - else response.content.encode(), + content=response.content if isinstance(response.content, bytes) else response.content.encode(), status=response.status_code, reason=response.reason, encoding=response.encoding or "utf-8", diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 79b52d1..72c39e9 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -49,9 +49,7 @@ class ResponseEncoding: @classmethod @lru_cache(maxsize=128) - def get_value( - cls, content_type: Optional[str], text: Optional[str] = "test" - ) -> str: + def get_value(cls, content_type: Optional[str], text: Optional[str] = "test") -> str: """Determine the appropriate character encoding from a content-type header. The encoding is determined by these rules in order: @@ -84,9 +82,7 @@ class ResponseEncoding: encoding = cls.__DEFAULT_ENCODING if encoding: - _ = text.encode( - encoding - ) # Validate encoding and validate it can encode the given text + _ = text.encode(encoding) # Validate encoding and validate it can encode the given text return encoding return cls.__DEFAULT_ENCODING @@ -129,9 +125,7 @@ class Response(Selector): **selector_config, ) # For easier debugging while working from a Python shell - log.info( - f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})" - ) + log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})") class BaseFetcher: @@ -190,18 +184,12 @@ class BaseFetcher: setattr(cls, key, value) else: # Yup, no fun allowed LOL - raise AttributeError( - f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?' - ) + raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') else: - raise ValueError( - f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?' - ) + raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') if not kwargs: - raise AttributeError( - f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?" - ) + raise AttributeError(f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?") @classmethod def _generate_parser_arguments(cls) -> Dict: @@ -217,9 +205,7 @@ class BaseFetcher: ) if cls.adaptive_domain: if not isinstance(cls.adaptive_domain, str): - log.warning( - '[Ignored] The argument "adaptive_domain" must be of string type' - ) + log.warning('[Ignored] The argument "adaptive_domain" must be of string type') else: parser_arguments.update({"adaptive_domain": cls.adaptive_domain}) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index ab91174..6d666a8 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -30,9 +30,7 @@ def intercept_route(route: Route): :return: PlayWright `Route` object """ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: - log.debug( - f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"' - ) + log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') route.abort() else: route.continue_() @@ -45,17 +43,13 @@ async def async_intercept_route(route: async_Route): :return: PlayWright `Route` object """ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: - log.debug( - f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"' - ) + log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') await route.abort() else: await route.continue_() -def construct_proxy_dict( - proxy_string: str | Dict[str, str], as_tuple=False -) -> Optional[Dict | Tuple]: +def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]: """Validate a proxy and return it in the acceptable format for Playwright Reference: https://playwright.dev/python/docs/network#http-proxy @@ -65,10 +59,7 @@ def construct_proxy_dict( """ if isinstance(proxy_string, str): proxy = urlparse(proxy_string) - if ( - proxy.scheme not in ("http", "https", "socks4", "socks5") - or not proxy.hostname - ): + if proxy.scheme not in ("http", "https", "socks4", "socks5") or not proxy.hostname: raise ValueError("Invalid proxy string!") try: diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index ea7f749..fc72de4 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -112,9 +112,7 @@ class StealthyFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") with StealthySession( wait=wait, @@ -210,9 +208,7 @@ class StealthyFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") async with AsyncStealthySession( wait=wait, @@ -318,9 +314,7 @@ class DynamicFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - raise ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") with DynamicSession( wait=wait, @@ -404,9 +398,7 @@ class DynamicFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - raise ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") async with AsyncDynamicSession( wait=wait, diff --git a/scrapling/parser.py b/scrapling/parser.py index 730f490..f943db8 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -110,22 +110,16 @@ class Selector(SelectorsGeneration): If empty, default values will be used. """ if root is None and content is None: - raise ValueError( - "Selector class needs HTML content, or root arguments to work" - ) + raise ValueError("Selector class needs HTML content, or root arguments to work") self.__text = None if root is None: if isinstance(content, str): - body = ( - content.strip().replace("\x00", "").encode(encoding) or b"" - ) + body = content.strip().replace("\x00", "").encode(encoding) or b"" elif isinstance(content, bytes): body = content.replace(b"\x00", b"").strip() else: - raise TypeError( - f"content argument must be str or bytes, got {type(content)}" - ) + raise TypeError(f"content argument must be str or bytes, got {type(content)}") # https://lxml.de/api/lxml.etree.HTMLParser-class.html parser = HTMLParser( @@ -165,16 +159,10 @@ class Selector(SelectorsGeneration): } if not hasattr(storage, "__wrapped__"): - raise ValueError( - "Storage class must be wrapped with lru_cache decorator, see docs for info" - ) + raise ValueError("Storage class must be wrapped with lru_cache decorator, see docs for info") - if not issubclass( - storage.__wrapped__, StorageSystemMixin - ): # pragma: no cover - raise ValueError( - "Storage system must be inherited from class `StorageSystemMixin`" - ) + if not issubclass(storage.__wrapped__, StorageSystemMixin): # pragma: no cover + raise ValueError("Storage system must be inherited from class `StorageSystemMixin`") self._storage = storage(**storage_args) @@ -239,9 +227,7 @@ class Selector(SelectorsGeneration): def __element_convertor(self, element: HtmlElement) -> "Selector": """Used internally to convert a single HtmlElement to Selector directly without checks""" - db_instance = ( - self._storage if (hasattr(self, "_storage") and self._storage) else None - ) + db_instance = self._storage if (hasattr(self, "_storage") and self._storage) else None return Selector( root=element, url=self.url, @@ -355,9 +341,7 @@ class Selector(SelectorsGeneration): @property def html_content(self) -> TextHandler: """Return the inner HTML code of the element""" - return TextHandler( - tostring(self._root, encoding="unicode", method="html", with_tail=False) - ) + return TextHandler(tostring(self._root, encoding="unicode", method="html", with_tail=False)) body = html_content @@ -404,9 +388,7 @@ class Selector(SelectorsGeneration): def siblings(self) -> "Selectors": """Return other children of the current element's parent or empty list otherwise""" if self.parent: - return Selectors( - child for child in self.parent.children if child._root != self._root - ) + return Selectors(child for child in self.parent.children if child._root != self._root) return Selectors() def iterancestors(self) -> Generator["Selector", None, None]: @@ -519,9 +501,7 @@ class Selector(SelectorsGeneration): log.debug(f"Highest probability was {highest_probability}%") log.debug("Top 5 best matching elements are: ") for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]: - log.debug( - f"{percent} -> {self.__handle_elements(score_table[percent])}" - ) + log.debug(f"{percent} -> {self.__handle_elements(score_table[percent])}") if not selector_type: return score_table[highest_probability] @@ -658,9 +638,7 @@ class Selector(SelectorsGeneration): SelectorError, SelectorSyntaxError, ) as e: - raise SelectorSyntaxError( - f"Invalid CSS selector '{selector}': {str(e)}" - ) from e + raise SelectorSyntaxError(f"Invalid CSS selector '{selector}': {str(e)}") from e def xpath( self, @@ -702,9 +680,7 @@ class Selector(SelectorsGeneration): elif self.__adaptive_enabled and auto_save: self.save(elements[0], identifier or selector) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) elif self.__adaptive_enabled: if adaptive: element_data = self.retrieve(identifier or selector) @@ -713,9 +689,7 @@ class Selector(SelectorsGeneration): if elements is not None and auto_save: self.save(elements[0], identifier or selector) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) else: if adaptive: log.warning( @@ -726,9 +700,7 @@ class Selector(SelectorsGeneration): "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info." ) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) except ( SelectorError, @@ -751,9 +723,7 @@ class Selector(SelectorsGeneration): """ if not args and not kwargs: - raise TypeError( - "You have to pass something to search with, like tag name(s), tag attributes, or both." - ) + raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.") attributes = dict() tags, patterns = set(), set() @@ -766,18 +736,11 @@ class Selector(SelectorsGeneration): elif type(arg) in (list, tuple, set): if not all(map(lambda x: isinstance(x, str), arg)): - raise TypeError( - "Nested Iterables are not accepted, only iterables of tag names are accepted" - ) + raise TypeError("Nested Iterables are not accepted, only iterables of tag names are accepted") tags.update(set(arg)) elif isinstance(arg, dict): - if not all( - [ - (isinstance(k, str) and isinstance(v, str)) - for k, v in arg.items() - ] - ): + if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in arg.items()]): raise TypeError( "Nested dictionaries are not accepted, only string keys and string values are accepted" ) @@ -795,13 +758,9 @@ class Selector(SelectorsGeneration): ) else: - raise TypeError( - f'Argument with type "{type(arg)}" is not accepted, please read the docs.' - ) + raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.') - if not all( - [(isinstance(k, str) and isinstance(v, str)) for k, v in kwargs.items()] - ): + if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in kwargs.items()]): raise TypeError("Only string values are accepted for arguments") for attribute_name, value in kwargs.items(): @@ -825,9 +784,7 @@ class Selector(SelectorsGeneration): if results: # From the results, get the ones that fulfill passed regex patterns for pattern in patterns: - results = results.filter( - lambda e: e.text.re(pattern, check_match=True) - ) + results = results.filter(lambda e: e.text.re(pattern, check_match=True)) # From the results, get the ones that fulfill passed functions for function in functions: @@ -858,9 +815,7 @@ class Selector(SelectorsGeneration): return element return None - def __calculate_similarity_score( - self, original: Dict, candidate: HtmlElement - ) -> float: + def __calculate_similarity_score(self, original: Dict, candidate: HtmlElement) -> float: """Used internally to calculate a score that shows how a candidate element similar to the original one :param original: The original element in the form of the dictionary generated from `element_to_dict` function @@ -877,15 +832,11 @@ class Selector(SelectorsGeneration): checks += 1 if original["text"]: - score += SequenceMatcher( - None, original["text"], candidate.get("text") or "" - ).ratio() # * 0.3 # 30% + score += SequenceMatcher(None, original["text"], candidate.get("text") or "").ratio() # * 0.3 # 30% checks += 1 # if both don't have attributes, it still counts for something! - score += self.__calculate_dict_diff( - original["attributes"], candidate["attributes"] - ) # * 0.3 # 30% + score += self.__calculate_dict_diff(original["attributes"], candidate["attributes"]) # * 0.3 # 30% checks += 1 # Separate similarity test for class, id, href,... this will help in full structural changes @@ -903,9 +854,7 @@ class Selector(SelectorsGeneration): ).ratio() # * 0.3 # 30% checks += 1 - score += SequenceMatcher( - None, original["path"], candidate["path"] - ).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["path"], candidate["path"]).ratio() # * 0.1 # 10% checks += 1 if original.get("parent_name"): @@ -944,14 +893,8 @@ class Selector(SelectorsGeneration): @staticmethod def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float: """Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries""" - score = ( - SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() - * 0.5 - ) - score += ( - SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() - * 0.5 - ) + score = SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() * 0.5 + score += SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() * 0.5 return score def save(self, element: Union["Selector", HtmlElement], identifier: str) -> None: @@ -1031,9 +974,7 @@ class Selector(SelectorsGeneration): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it """ - return self.text.re_first( - regex, default, replace_entities, clean_match, case_sensitive - ) + return self.text.re_first(regex, default, replace_entities, clean_match, case_sensitive) @staticmethod def __get_attributes(element: HtmlElement, ignore_attributes: List | Tuple) -> Dict: @@ -1052,9 +993,7 @@ class Selector(SelectorsGeneration): """Calculate a score of how much these elements are alike and return True if the score is higher or equals the threshold""" candidate_attributes = ( - self.__get_attributes(candidate, ignore_attributes) - if ignore_attributes - else candidate.attrib + self.__get_attributes(candidate, ignore_attributes) if ignore_attributes else candidate.attrib ) score, checks = 0, 0 @@ -1116,11 +1055,7 @@ class Selector(SelectorsGeneration): similar_elements = list() current_depth = len(list(root.iterancestors())) - target_attrs = ( - self.__get_attributes(root, ignore_attributes) - if ignore_attributes - else root.attrib - ) + target_attrs = self.__get_attributes(root, ignore_attributes) if ignore_attributes else root.attrib path_parts = [self.tag] if (parent := root.getparent()) is not None: @@ -1129,9 +1064,7 @@ class Selector(SelectorsGeneration): path_parts.insert(0, grandparent.tag) xpath_path = "//{}".format("/".join(path_parts)) - potential_matches = root.xpath( - f"{xpath_path}[count(ancestor::*) = {current_depth}]" - ) + potential_matches = root.xpath(f"{xpath_path}[count(ancestor::*) = {current_depth}]") for potential_match in potential_matches: if potential_match != root and self.__are_alike( @@ -1275,12 +1208,7 @@ class Selectors(List[Selector]): :return: `Selectors` class. """ - results = [ - n.xpath( - selector, identifier or selector, False, auto_save, percentage, **kwargs - ) - for n in self - ] + results = [n.xpath(selector, identifier or selector, False, auto_save, percentage, **kwargs) for n in self] return self.__class__(flatten(results)) def css( @@ -1308,10 +1236,7 @@ class Selectors(List[Selector]): :return: `Selectors` class. """ - results = [ - n.css(selector, identifier or selector, False, auto_save, percentage) - for n in self - ] + results = [n.css(selector, identifier or selector, False, auto_save, percentage) for n in self] return self.__class__(flatten(results)) def re( @@ -1329,10 +1254,7 @@ class Selectors(List[Selector]): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it """ - results = [ - n.text.re(regex, replace_entities, clean_match, case_sensitive) - for n in self - ] + results = [n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self] return TextHandlers(flatten(results)) def re_first( From aaec00563c9de22a771d29c3c160c2f6ba27bce2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:34:28 +0300 Subject: [PATCH 17/35] ops: update pre-commit hooks --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e4885b..a67bd50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.8.3 + rev: 1.8.6 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.11.5 + rev: v0.13.0 hooks: # Run the linter. - id: ruff From 8c4d6158d6d69e71e1e306ab9c2fe1c7378558a7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:59:12 +0300 Subject: [PATCH 18/35] feat(browser fetchers): A new option to control page JS --- scrapling/engines/_browsers/_base.py | 2 ++ scrapling/engines/_browsers/_camoufox.py | 31 +++++++++++++++++---- scrapling/engines/_browsers/_controllers.py | 25 ++++++++++++++--- scrapling/engines/_browsers/_validators.py | 2 ++ scrapling/fetchers.py | 14 +++++++++- 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6412698..5bb86ab 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -162,6 +162,7 @@ class DynamicSessionMixin: self.disable_resources = config.disable_resources self.cdp_url = config.cdp_url self.network_idle = config.network_idle + self.load_dom = config.load_dom self.wait_selector = config.wait_selector self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state @@ -216,6 +217,7 @@ class StealthySessionMixin: self.block_webrtc = config.block_webrtc self.allow_webgl = config.allow_webgl self.network_idle = config.network_idle + self.load_dom = config.load_dom self.humanize = config.humanize self.solve_cloudflare = config.solve_cloudflare self.wait = config.wait diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index fcb77ac..31113be 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -46,6 +46,7 @@ class StealthySession(StealthySessionMixin, SyncSession): "block_webrtc", "allow_webgl", "network_idle", + "load_dom", "humanize", "solve_cloudflare", "wait", @@ -82,6 +83,7 @@ class StealthySession(StealthySessionMixin, SyncSession): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -116,6 +118,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -142,6 +145,7 @@ class StealthySession(StealthySessionMixin, SyncSession): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, max_pages=__max_pages, disable_ads=disable_ads, allow_webgl=allow_webgl, @@ -259,6 +263,7 @@ class StealthySession(StealthySessionMixin, SyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, solve_cloudflare: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: @@ -276,6 +281,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. @@ -292,6 +298,7 @@ class StealthySession(StealthySessionMixin, SyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), @@ -321,7 +328,8 @@ class StealthySession(StealthySessionMixin, SyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -333,7 +341,8 @@ class StealthySession(StealthySessionMixin, SyncSession): self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -349,7 +358,8 @@ class StealthySession(StealthySessionMixin, SyncSession): waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: @@ -382,6 +392,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -416,6 +427,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -441,6 +453,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): timeout=timeout, cookies=cookies, headless=headless, + load_dom=load_dom, humanize=humanize, max_pages=max_pages, disable_ads=disable_ads, @@ -559,6 +572,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, solve_cloudflare: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: @@ -576,6 +590,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. @@ -591,6 +606,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), @@ -620,7 +636,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -632,7 +649,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -648,7 +666,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index a53fa62..96d7291 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -54,6 +54,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): "cookies", "disable_resources", "network_idle", + "load_dom", "wait_selector", "init_script", "wait_selector_state", @@ -93,6 +94,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -116,6 +118,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ @@ -130,6 +133,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): stealth=stealth, cdp_url=cdp_url, cookies=cookies, + load_dom=load_dom, headless=headless, useragent=useragent, max_pages=__max_pages, @@ -208,6 +212,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -224,6 +229,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ @@ -239,6 +245,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), PlaywrightConfig, @@ -267,7 +274,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -287,7 +295,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession): waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover @@ -335,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -347,6 +357,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -374,6 +385,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): stealth=stealth, cdp_url=cdp_url, cookies=cookies, + load_dom=load_dom, headless=headless, useragent=useragent, max_pages=max_pages, @@ -453,6 +465,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -469,6 +482,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ @@ -484,6 +498,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), PlaywrightConfig, @@ -512,7 +527,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -532,7 +548,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index c34d6fe..a20ae55 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -35,6 +35,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): wait_selector: Optional[str] = None cookies: Optional[List[Dict]] = None network_idle: bool = False + load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" selector_config: Optional[Dict] = None @@ -92,6 +93,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): block_webrtc: bool = False allow_webgl: bool = True network_idle: bool = False + load_dom: bool = True humanize: bool | float = True solve_cloudflare: bool = False wait: int | float = 0 diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index fc72de4..2421b42 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -56,6 +56,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -92,6 +93,7 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -123,6 +125,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -152,6 +155,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -188,6 +192,7 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -220,6 +225,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -280,6 +286,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -293,6 +300,7 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -325,6 +333,7 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, real_chrome=real_chrome, page_action=page_action, @@ -364,6 +373,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -377,6 +387,7 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -402,6 +413,7 @@ class DynamicFetcher(BaseFetcher): async with AsyncDynamicSession( wait=wait, + max_pages=1, proxy=proxy, locale=locale, timeout=timeout, @@ -409,8 +421,8 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, - max_pages=1, real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, From 80077953f60914ad5c0f770b011575076c4d21fb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 04:19:16 +0300 Subject: [PATCH 19/35] docs: update all pages related to last changes --- docs/fetching/dynamic.md | 16 +++++++--------- docs/fetching/stealthy.md | 16 +++++++--------- docs/overview.md | 2 -- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 29bf1d6..c29bdf4 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -14,10 +14,7 @@ Check out how to configure the parsing options [here](choosing.md#parser-configu Now, we will review most of the arguments one by one, using examples. If you want to jump to a table of all arguments for quick reference, [click here](#full-list-of-arguments) -> Notes: -> -> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state). -> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. +> Note: The async version of the `fetch` method is the `async_fetch` method, of course. This fetcher currently provides four main run options, which can be mixed as desired. @@ -75,6 +72,7 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | cookies | Set cookies for the next request. | ✔️ | | useragent | Pass a useragent string to be used. **Otherwise, the fetcher will generate and use a real Useragent of the same browser.** | ✔️ | | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ | | timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | | page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ | @@ -167,7 +165,7 @@ page = DynamicFetcher.fetch( ``` This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. -After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. +After that, if `load_dom` is enabled (the default), the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): @@ -271,14 +269,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit of the maximum number of pages allowed and with each request, the library will close all tabs that finished its task and check if the number of the current tabs is lower than the number of maximum allowed number of pages/tabs then: +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: -1. If you are within the allowed range, the fetcher will create a new tab for you and then all is as normal. -2. Otherwise, it will keep checking every sub second if creating a new tab is allowed or not for 60 seconds then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) -In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time but this logic proved to have flaws since it's nearly impossible to protections pages/tabs from contamination of the previous configuration you used with the request before this one. +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws since it's nearly impossible to protect pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index f090aa5..61febeb 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -12,10 +12,7 @@ You have one primary way to import this Fetcher, which is the same for all fetch ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) -> Notes: -> -> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state). -> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. +> Note: The async version of the `fetch` method is the `async_fetch` method, of course. ## Full list of arguments Before jumping to [examples](#examples), here's the full list of arguments @@ -40,6 +37,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | disable_ads | Disabled by default; this installs the `uBlock Origin` addon on the browser if enabled. | ✔️ | | solve_cloudflare | When enabled, fetcher solves all three types of Cloudflare's Turnstile wait/captcha page before returning the response to you. | ✔️ | | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ | | timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | @@ -188,7 +186,7 @@ page = StealthyFetcher.fetch( ``` This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. -After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. +After that, if `load_dom` is enabled (the default), the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): @@ -276,14 +274,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit of the maximum number of pages allowed and with each request, the library will close all tabs that finished its task and check if the number of the current tabs is lower than the number of maximum allowed number of pages/tabs then: +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: -1. If you are within the allowed range, the fetcher will create a new tab for you and then all is as normal. -2. Otherwise, it will keep checking every sub second if creating a new tab is allowed or not for 60 seconds then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) -In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time but this logic proved to have flaws since it's nearly impossible to protections pages/tabs from contamination of the previous configuration you used with the request before this one. +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws since it's nearly impossible to protect pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/docs/overview.md b/docs/overview.md index acde1b3..561e10b 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -292,7 +292,6 @@ It's built on top of [Playwright](https://playwright.dev/python/) and it's curre - Stealthy Playwright with custom stealth mode explicitly written for it. It's not top-tier stealth mode, but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/). Check out the `StealthyFetcher` class below for more advanced stealth mode. It uses the Chromium browser. - Real browsers like your Chrome browser by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it. -> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later. Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments. @@ -314,7 +313,6 @@ True >>> page.status == 200 True ``` -> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later. Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments. From 521366e77e8689a39884b58c2b08070a05f4b618 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 04:29:09 +0300 Subject: [PATCH 20/35] build: bump up deps --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad4550f..37c15d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "lxml>=6.0.0", + "lxml>=6.0.1", "cssselect>=1.3.0", - "click>=8.2.1", - "orjson>=3.11.2", + "click>=8.2.2", + "orjson>=3.11.3", "tldextract>=5.3.0", "curl_cffi>=0.13.0", "playwright>=1.52.0", @@ -71,7 +71,7 @@ dependencies = [ [project.optional-dependencies] ai = [ - "mcp>=1.13.0", + "mcp>=1.14.0", "markdownify>=1.2.0", ] shell = [ From d7e3deae2a8284d57bde92a8b5279e7ec1dc944e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 05:10:56 +0300 Subject: [PATCH 21/35] fix: fix invalid dep version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 37c15d6..a1aa1af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ classifiers = [ dependencies = [ "lxml>=6.0.1", "cssselect>=1.3.0", - "click>=8.2.2", + "click>=8.2.1", "orjson>=3.11.3", "tldextract>=5.3.0", "curl_cffi>=0.13.0", From 13d7e70cb70e3af4bfd6b22b5e5e72facd454ad3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 16:07:48 +0300 Subject: [PATCH 22/35] refactor: Make all fetchers as an optional dependency group + Removing some dead code --- pyproject.toml | 8 +- scrapling/cli.py | 20 +++- scrapling/core/shell.py | 98 +++++++------------- scrapling/core/translator.py | 4 +- scrapling/core/utils/__init__.py | 10 ++ scrapling/core/utils/_shell.py | 48 ++++++++++ scrapling/core/{utils.py => utils/_utils.py} | 0 scrapling/engines/__init__.py | 16 ---- scrapling/engines/_browsers/_base.py | 15 ++- scrapling/engines/_browsers/_camoufox.py | 4 +- scrapling/engines/_browsers/_config_tools.py | 3 +- scrapling/engines/_browsers/_controllers.py | 4 +- scrapling/engines/_browsers/_validators.py | 2 +- scrapling/engines/static.py | 4 +- scrapling/engines/toolbelt/__init__.py | 9 -- scrapling/engines/toolbelt/custom.py | 3 +- scrapling/engines/toolbelt/fingerprints.py | 4 +- scrapling/engines/toolbelt/navigation.py | 45 --------- scrapling/fetchers.py | 12 ++- scrapling/parser.py | 7 +- 20 files changed, 142 insertions(+), 174 deletions(-) create mode 100644 scrapling/core/utils/__init__.py create mode 100644 scrapling/core/utils/_shell.py rename scrapling/core/{utils.py => utils/_utils.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index a1aa1af..5c708f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,10 @@ dependencies = [ "click>=8.2.1", "orjson>=3.11.3", "tldextract>=5.3.0", +] + +[project.optional-dependencies] +fetchers = [ "curl_cffi>=0.13.0", "playwright>=1.52.0", "rebrowser-playwright>=1.52.0", @@ -68,15 +72,15 @@ dependencies = [ "geoip2>=5.1.0", "msgspec>=0.19.0", ] - -[project.optional-dependencies] ai = [ "mcp>=1.14.0", "markdownify>=1.2.0", + "scrapling[fetchers]", ] shell = [ "IPython>=8.37", # The last version that supports Python 3.10 "markdownify>=1.2.0", + "scrapling[fetchers]", ] all = [ "scrapling[ai,shell]", diff --git a/scrapling/cli.py b/scrapling/cli.py index e4b3c7f..3e052ac 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,11 +2,9 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable -from scrapling.core.utils import log -from scrapling.engines.toolbelt import Response +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _CookieParser, _ParseHeaders from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable -from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher -from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders from orjson import loads as json_loads, JSONDecodeError from click import command, option, Choice, group, argument @@ -40,6 +38,8 @@ def __Request_and_Save( **kwargs, ) -> None: """Make a request using the specified fetcher function and save the result""" + from scrapling.core.shell import Convertor + # Handle relative paths - convert to an absolute path based on the current working directory output_path = Path(output_file) if not output_path.is_absolute(): @@ -251,6 +251,8 @@ def get( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs) @@ -347,6 +349,8 @@ def post( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs) @@ -439,6 +443,8 @@ def put( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs) @@ -524,6 +530,8 @@ def delete( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs) @@ -643,6 +651,8 @@ def fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import DynamicFetcher + __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) @@ -790,6 +800,8 @@ def stealthy_fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import StealthyFetcher + __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 8459a60..2b23b2b 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -2,7 +2,6 @@ from re import sub as re_sub from sys import stderr from functools import wraps -from http import cookies as Cookie from collections import namedtuple from shlex import split as shlex_split from tempfile import mkstemp as make_temp_file @@ -23,25 +22,17 @@ from logging import ( from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ -from scrapling.core.custom_types import TextHandler -from scrapling.core.utils import log from scrapling.parser import Selector, Selectors +from scrapling.core.custom_types import TextHandler +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _ParseHeaders, _CookieParser from scrapling.core._types import ( - List, Optional, Dict, - Tuple, Any, extraction_types, Generator, ) -from scrapling.fetchers import ( - Fetcher, - AsyncFetcher, - DynamicFetcher, - StealthyFetcher, - Response, -) _known_logging_levels = { @@ -71,46 +62,6 @@ Request = namedtuple( ) -def _CookieParser(cookie_string): - # Errors will be handled on call so the log can be specified - cookie_parser = Cookie.SimpleCookie() - cookie_parser.load(cookie_string) - for key, morsel in cookie_parser.items(): - yield key, morsel.value - - -def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: - """Parses headers into separate header and cookie dictionaries.""" - header_dict = dict() - cookie_dict = dict() - - for header_line in header_lines: - if ":" not in header_line: - if header_line.endswith(";"): - header_key = header_line[:-1].strip() - header_value = "" - header_dict[header_key] = header_value - else: - raise ValueError(f"Could not parse header without colon: '{header_line}'.") - else: - header_key, header_value = header_line.split(":", 1) - header_key = header_key.strip() - header_value = header_value.strip() - - if parse_cookies: - if header_key.lower() == "cookie": - try: - cookie_dict = {key: value for key, value in _CookieParser(header_value)} - except Exception as e: # pragma: no cover - raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") - else: - header_dict[header_key] = header_value - else: - header_dict[header_key] = header_value - - return header_dict, cookie_dict - - # Suppress exit on error to handle parsing errors gracefully class NoExitArgumentParser(ArgumentParser): # pragma: no cover def error(self, message): @@ -128,6 +79,9 @@ class CurlParser: """Builds the argument parser for relevant curl flags from DevTools.""" def __init__(self): + from scrapling.fetchers import Fetcher as __Fetcher + + self.__fetcher = __Fetcher # We will use argparse parser to parse the curl command directly instead of regex # We will focus more on flags that will show up on curl commands copied from DevTools's network tab _parser = NoExitArgumentParser(add_help=False) # Disable default help @@ -343,7 +297,7 @@ class CurlParser: _ = request_args.pop("json", None) try: - return getattr(Fetcher, method)(**request_args) + return getattr(self.__Fetcher, method)(**request_args) except Exception as e: # pragma: no cover log.error(f"Error calling Fetcher.{method}: {e}") return None @@ -377,6 +331,19 @@ class CustomShell: """A custom IPython shell with minimal dependencies""" def __init__(self, code, log_level="debug"): + from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed + from scrapling.fetchers import ( + Fetcher as __Fetcher, + AsyncFetcher as __AsyncFetcher, + DynamicFetcher as __DynamicFetcher, + StealthyFetcher as __StealthyFetcher, + ) + + self.__InteractiveShellEmbed = __InteractiveShellEmbed + self.__Fetcher = __Fetcher + self.__AsyncFetcher = __AsyncFetcher + self.__DynamicFetcher = __DynamicFetcher + self.__StealthyFetcher = __StealthyFetcher self.code = code self.page = None self.pages = Selectors([]) @@ -400,7 +367,7 @@ class CustomShell: if self.log_level: getLogger("scrapling").setLevel(self.log_level) - settings = Fetcher.display_config() + settings = self.__Fetcher.display_config() settings.pop("storage", None) settings.pop("storage_args", None) log.info(f"Scrapling {__version__} shell started") @@ -466,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit. """Create a namespace with application-specific objects""" # Create wrapped versions of fetch functions - get = self.create_wrapper(Fetcher.get) - post = self.create_wrapper(Fetcher.post) - put = self.create_wrapper(Fetcher.put) - delete = self.create_wrapper(Fetcher.delete) - dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch) - stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch) + get = self.create_wrapper(self.__Fetcher.get) + post = self.create_wrapper(self.__Fetcher.post) + put = self.create_wrapper(self.__Fetcher.put) + delete = self.create_wrapper(self.__Fetcher.delete) + dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) + stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher) # Create the namespace dictionary @@ -480,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit. "post": post, "put": put, "delete": delete, - "Fetcher": Fetcher, - "AsyncFetcher": AsyncFetcher, + "Fetcher": self.__Fetcher, + "AsyncFetcher": self.__AsyncFetcher, "fetch": dynamic_fetch, - "DynamicFetcher": DynamicFetcher, + "DynamicFetcher": self.__DynamicFetcher, "stealthy_fetch": stealthy_fetch, - "StealthyFetcher": StealthyFetcher, + "StealthyFetcher": self.__StealthyFetcher, "Selector": Selector, "page": self.page, "response": self.page, @@ -502,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit. def start(self): # pragma: no cover """Start the interactive shell""" - from IPython.terminal.embed import InteractiveShellEmbed # Get our namespace with application objects namespace = self.get_namespace() - ipython_shell = InteractiveShellEmbed( + ipython_shell = self.__InteractiveShellEmbed( banner1=self.banner(), banner2="", enable_tip=False, diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 88dfbff..d98092e 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -10,10 +10,10 @@ So you don't have to learn a new selectors/api method like what bs4 done with so from functools import lru_cache -from cssselect import HTMLTranslator as OriginalHTMLTranslator -from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from cssselect.xpath import ExpressionError from cssselect.xpath import XPathExpr as OriginalXPathExpr +from cssselect import HTMLTranslator as OriginalHTMLTranslator +from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from scrapling.core._types import Any, Optional, Protocol, Self diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.py new file mode 100644 index 0000000..dc95705 --- /dev/null +++ b/scrapling/core/utils/__init__.py @@ -0,0 +1,10 @@ +from ._utils import ( + log, + __CONSECUTIVE_SPACES_REGEX__, + flatten, + _is_iterable, + _StorageTools, + clean_spaces, + html_forbidden, +) +from ._shell import _CookieParser, _ParseHeaders diff --git a/scrapling/core/utils/_shell.py b/scrapling/core/utils/_shell.py new file mode 100644 index 0000000..05420ae --- /dev/null +++ b/scrapling/core/utils/_shell.py @@ -0,0 +1,48 @@ +from http import cookies as Cookie + + +from scrapling.core._types import ( + List, + Dict, + Tuple, +) + + +def _CookieParser(cookie_string): + # Errors will be handled on call so the log can be specified + cookie_parser = Cookie.SimpleCookie() + cookie_parser.load(cookie_string) + for key, morsel in cookie_parser.items(): + yield key, morsel.value + + +def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: + """Parses headers into separate header and cookie dictionaries.""" + header_dict = dict() + cookie_dict = dict() + + for header_line in header_lines: + if ":" not in header_line: + if header_line.endswith(";"): + header_key = header_line[:-1].strip() + header_value = "" + header_dict[header_key] = header_value + else: + raise ValueError(f"Could not parse header without colon: '{header_line}'.") + else: + header_key, header_value = header_line.split(":", 1) + header_key = header_key.strip() + header_value = header_value.strip() + + if parse_cookies: + if header_key.lower() == "cookie": + try: + cookie_dict = {key: value for key, value in _CookieParser(header_value)} + except Exception as e: # pragma: no cover + raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") + else: + header_dict[header_key] = header_value + else: + header_dict[header_key] = header_value + + return header_dict, cookie_dict diff --git a/scrapling/core/utils.py b/scrapling/core/utils/_utils.py similarity index 100% rename from scrapling/core/utils.py rename to scrapling/core/utils/_utils.py diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index 477bb21..e69de29 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,16 +0,0 @@ -from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS -from .static import FetcherSession, FetcherClient, AsyncFetcherClient -from ._browsers import ( - DynamicSession, - AsyncDynamicSession, - StealthySession, - AsyncStealthySession, -) - -__all__ = [ - "FetcherSession", - "DynamicSession", - "AsyncDynamicSession", - "StealthySession", - "AsyncStealthySession", -] diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 5bb86ab..00c5567 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -12,20 +12,17 @@ from camoufox.utils import ( installed_verstr as camoufox_version, ) -from scrapling.engines.toolbelt import ( - intercept_route, - async_intercept_route, - get_os_name, -) -from ._page import PageInfo, PagePool -from ._config_tools import _compiled_stealth_scripts -from ._validators import validate, PlaywrightConfig, CamoufoxConfig -from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route from scrapling.core._types import ( Any, Dict, Optional, ) +from ._page import PageInfo, PagePool +from ._config_tools import _compiled_stealth_scripts +from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.fingerprints import get_os_name +from ._validators import validate, PlaywrightConfig, CamoufoxConfig __ff_version_str__ = camoufox_version().split(".", 1)[0] diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 31113be..6e0bc1e 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -25,11 +25,11 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - generate_convincing_referer, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") _UNSET = object() diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 96b7f41..4b405e6 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -6,7 +6,8 @@ from scrapling.engines.constants import ( HARMFUL_DEFAULT_ARGS, DEFAULT_FLAGS, ) -from scrapling.engines.toolbelt import js_bypass_path, generate_headers +from scrapling.engines.toolbelt.navigation import js_bypass_path +from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 96d7291..07efee2 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -26,11 +26,11 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - generate_convincing_referer, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer _UNSET = object() diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index a20ae55..ca64942 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -9,7 +9,7 @@ from scrapling.core._types import ( List, SelectorWaitStates, ) -from scrapling.engines.toolbelt import construct_proxy_dict +from scrapling.engines.toolbelt.navigation import construct_proxy_dict class PlaywrightConfig(Struct, kw_only=True, frozen=False): diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index e028feb..35271b8 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -26,11 +26,11 @@ from scrapling.core._types import ( from .toolbelt import ( Response, - generate_convincing_referer, generate_headers, - ResponseFactory, __default_useragent__, ) +from .toolbelt.convertor import ResponseFactory +from .toolbelt.fingerprints import generate_convincing_referer _UNSET = object() diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index d58fd57..04f43b1 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -5,16 +5,7 @@ from .custom import ( get_variable_name, ) from .fingerprints import ( - generate_convincing_referer, generate_headers, get_os_name, __default_useragent__, ) -from .navigation import ( - async_intercept_route, - construct_cdp_url, - construct_proxy_dict, - intercept_route, - js_bypass_path, -) -from .convertor import ResponseFactory diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 72c39e9..f9effa2 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -2,8 +2,10 @@ Functions related to custom types or type checking """ +from functools import lru_cache from email.message import Message +from scrapling.core.utils import log from scrapling.core._types import ( Any, Dict, @@ -12,7 +14,6 @@ from scrapling.core._types import ( Tuple, ) from scrapling.core.custom_types import MappingProxyType -from scrapling.core.utils import log, lru_cache from scrapling.parser import Selector, SQLiteStorageSystem diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 4ca8e38..bd836e7 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -2,13 +2,13 @@ Functions related to generating headers and fingerprints generally """ +from functools import lru_cache from platform import system as platform_system from tldextract import extract from browserforge.headers import Browser, HeaderGenerator from scrapling.core._types import Dict, Optional -from scrapling.core.utils import lru_cache __OS_NAME__ = platform_system() @@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]: "Linux": "linux", "Darwin": "macos", "Windows": "windows", - # For the future? because why not? - "iOS": "ios", }.get(__OS_NAME__) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 6d666a8..f2f445c 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> return None -def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: - """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists - - :param cdp_url: The target URL. - :param query_params: A dictionary of the parameters to add. - :return: The new CDP URL. - """ - try: - # Validate the base URL structure - parsed = urlparse(cdp_url) - - # Check scheme - if parsed.scheme not in ("ws", "wss"): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") - - # Validate hostname and port - if not parsed.netloc: - raise ValueError("Invalid hostname for the CDP URL") - - try: - # Checking if the port is valid (if available) - _ = parsed.port - except ValueError: - # urlparse will raise `ValueError` if the port can't be casted to integer - raise ValueError("Invalid port for the CDP URL") - - # Ensure the path starts with / - path = parsed.path - if not path.startswith("/"): - path = "/" + path - - # Reconstruct the base URL with validated parts - validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" - - # Add query parameters - if query_params: - query_string = urlencode(query_params) - return f"{validated_base}?{query_string}" - - return validated_base - - except Exception as e: - raise ValueError(f"Invalid CDP URL: {str(e)}") - - @lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 2421b42..323293d 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -6,15 +6,17 @@ from scrapling.core._types import ( SelectorWaitStates, Iterable, ) -from scrapling.engines import ( +from scrapling.engines.static import ( FetcherSession, - StealthySession, - AsyncStealthySession, - DynamicSession, - AsyncDynamicSession, FetcherClient as _FetcherClient, AsyncFetcherClient as _AsyncFetcherClient, ) +from scrapling.engines._browsers import ( + DynamicSession, + StealthySession, + AsyncDynamicSession, + AsyncStealthySession, +) from scrapling.engines.toolbelt import BaseFetcher, Response __FetcherClientInstance__ = _FetcherClient() diff --git a/scrapling/parser.py b/scrapling/parser.py index f943db8..5335a82 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,12 +1,11 @@ -from pathlib import Path import re +from pathlib import Path from inspect import signature -from difflib import SequenceMatcher from urllib.parse import urljoin +from difflib import SequenceMatcher -from cssselect import SelectorError, SelectorSyntaxError -from cssselect import parse as split_selectors from lxml.html import HtmlElement, HtmlMixin, HTMLParser +from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors from lxml.etree import ( XPath, tostring, From d6844232eb6d452dce763d79d7956a5a0942dea8 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 16:07:54 +0300 Subject: [PATCH 23/35] tests: Update all tests according to new update --- tests/fetchers/async/test_camoufox_session.py | 2 +- tests/fetchers/async/test_dynamic_session.py | 2 +- tests/fetchers/test_response_handling.py | 2 +- tests/fetchers/test_utils.py | 38 ------------------- 4 files changed, 3 insertions(+), 41 deletions(-) diff --git a/tests/fetchers/async/test_camoufox_session.py b/tests/fetchers/async/test_camoufox_session.py index a2e0075..2971488 100644 --- a/tests/fetchers/async/test_camoufox_session.py +++ b/tests/fetchers/async/test_camoufox_session.py @@ -4,7 +4,7 @@ import asyncio import pytest_httpbin -from scrapling.engines import AsyncStealthySession +from scrapling.engines._browsers import AsyncStealthySession @pytest_httpbin.use_class_based_httpbin diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py index 24c6860..d7a4ea9 100644 --- a/tests/fetchers/async/test_dynamic_session.py +++ b/tests/fetchers/async/test_dynamic_session.py @@ -3,7 +3,7 @@ import asyncio import pytest_httpbin -from scrapling.engines import AsyncDynamicSession +from scrapling.engines._browsers import AsyncDynamicSession @pytest_httpbin.use_class_based_httpbin diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py index 7ff0f18..4db96bf 100644 --- a/tests/fetchers/test_response_handling.py +++ b/tests/fetchers/test_response_handling.py @@ -1,8 +1,8 @@ from unittest.mock import Mock from scrapling.parser import Selector -from scrapling.engines.toolbelt import ResponseFactory, Response from scrapling.engines.toolbelt.custom import ResponseEncoding +from scrapling.engines.toolbelt.convertor import ResponseFactory, Response class TestResponseFactory: diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index b787d36..4a1563c 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -4,7 +4,6 @@ from pathlib import Path from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, - construct_cdp_url, js_bypass_path ) from scrapling.engines.toolbelt.fingerprints import ( @@ -216,43 +215,6 @@ class TestConstructProxyDict: construct_proxy_dict({"invalid": "structure"}) -class TestConstructCdpUrl: - """Test CDP URL construction""" - - def test_basic_cdp_url(self): - """Test basic CDP URL""" - result = construct_cdp_url("ws://localhost:9222/devtools/browser") - assert result == "ws://localhost:9222/devtools/browser" - - def test_cdp_url_with_params(self): - """Test CDP URL with query parameters""" - params = {"timeout": "30000", "headless": "true"} - result = construct_cdp_url("ws://localhost:9222/devtools/browser", params) - - assert "timeout=30000" in result - assert "headless=true" in result - - def test_cdp_url_without_leading_slash(self): - """Test CDP URL without a leading slash in the path""" - with pytest.raises(ValueError): - construct_cdp_url("ws://localhost:9222devtools/browser") - - def test_invalid_cdp_scheme(self): - """Test invalid CDP URL scheme""" - with pytest.raises(ValueError): - construct_cdp_url("http://localhost:9222/devtools/browser") - - def test_invalid_cdp_netloc(self): - """Test invalid CDP URL network location""" - with pytest.raises(ValueError): - construct_cdp_url("ws:///devtools/browser") - - def test_malformed_cdp_url(self): - """Test malformed CDP URL""" - with pytest.raises(ValueError): - construct_cdp_url("not-a-url") - - class TestJsBypassPath: """Test JavaScript bypass path utility""" From f604485a099655c21de147d68eedf2ccf1db11d2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 04:45:56 +0300 Subject: [PATCH 24/35] docs: Update all pages related to last changes --- README.md | 52 +++++++++++++++++++++------------------ docs/fetching/dynamic.md | 5 +++- docs/fetching/stealthy.md | 3 ++- docs/index.md | 44 ++++++++++++++++++--------------- 4 files changed, 58 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 912350e..88ba6f8 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet - 🔄 **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms. - 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more. - 🔍 **Find Similar Elements**: Automatically locate elements similar to found elements. -- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage. +- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage. ([demo video](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) ### High-Performance & battle-tested Architecture - 🚀 **Lightning Fast**: Optimized performance outperforming most Python scraping libraries. @@ -134,7 +134,7 @@ quotes = page.css('.quote .text::text') # Advanced stealth mode (Keep the browser open until you finish) with StealthySession(headless=True, solve_cloudflare=True) as session: - page = session.fetch('https://nopecha.com/demo/cloudflare') + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) data = page.css('#padded_content a') # Or use one-off request style, it opens the browser for this request, then closes it after finishing @@ -143,7 +143,7 @@ data = page.css('#padded_content a') # Full browser automation (Keep the browser open until you finish) with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session: - page = session.fetch('https://quotes.toscrape.com/') + page = session.fetch('https://quotes.toscrape.com/', load_dom=False) data = page.xpath('//span[@class="text"]/text()') # XPath selector if you prefer it # Or use one-off request style, it opens the browser for this request, then closes it after finishing @@ -187,7 +187,7 @@ from scrapling.parser import Selector page = Selector("...") ``` -And it works exactly the same way! +And it works precisely the same way! ### Async Session Management Examples ```python @@ -271,29 +271,33 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -#### Fetchers Setup - -If you are going to use any of the fetchers or their classes, then install browser dependencies with -```bash -scrapling install -``` - -This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers. ### Optional Dependencies -- Install the MCP server feature: -```bash -pip install "scrapling[ai]" -``` -- Install shell features (Web Scraping shell and the `extract` command): -```bash -pip install "scrapling[shell]" -``` -- Install everything: -```bash -pip install "scrapling[all]" -``` +1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies, and then install their browser dependencies with + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. + +2. Extra features: + - Install the MCP server feature: + ```bash + pip install "scrapling[ai]" + ``` + - Install shell features (Web Scraping shell and the `extract` command): + ```bash + pip install "scrapling[shell]" + ``` + - Install everything: + ```bash + pip install "scrapling[all]" + ``` + Don't forget that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) ## Contributing diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index c29bdf4..11b5e41 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -62,7 +62,7 @@ DynamicFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222') Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). ## Full list of arguments -Scrapling provides many options with this fetcher. To make it as simple as possible, we will list the options here and give examples of using most of them. +Scrapling provides many options with this fetcher and its session classes. To make it as simple as possible, we will list the options here and give examples of using most of them. | Argument | Description | Optional | |:-------------------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| @@ -90,6 +90,9 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | +In the session classes, all these arguments can be set for the session globally. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, and `selector_config`. + + ## Examples It's easier to understand with examples, so let's take a look. diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 61febeb..d998246 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -15,7 +15,7 @@ Check out how to configure the parsing options [here](choosing.md#parser-configu > Note: The async version of the `fetch` method is the `async_fetch` method, of course. ## Full list of arguments -Before jumping to [examples](#examples), here's the full list of arguments +Scrapling provides many options with this fetcher and its session classes. Before jumping to the [examples](#examples), here's the full list of arguments | Argument | Description | Optional | @@ -47,6 +47,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | additional_args | Additional arguments to be passed to Camoufox as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | +In the session classes, all these arguments can be set for the session globally. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, and `selector_config`. ## Examples It's easier to understand with examples, so we will now review most of the arguments individually with examples. diff --git a/docs/index.md b/docs/index.md index 75accc9..ea7a2de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -114,29 +114,33 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -#### Fetchers Setup - -If you are going to use any of the fetchers or their session classes, then install browser dependencies with -```bash -scrapling install -``` - -This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers. ### Optional Dependencies -- Install the MCP server feature: -```bash -pip install "scrapling[ai]" -``` -- Install shell features (Web Scraping shell and the `extract` command): -```bash -pip install "scrapling[shell]" -``` -- Install everything: -```bash -pip install "scrapling[all]" -``` +1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies, and then install their browser dependencies with + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. + +2. Extra features: + - Install the MCP server feature: + ```bash + pip install "scrapling[ai]" + ``` + - Install shell features (Web Scraping shell and the `extract` command): + ```bash + pip install "scrapling[shell]" + ``` + - Install everything: + ```bash + pip install "scrapling[all]" + ``` + Don't forget that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) ## How the documentation is organized Scrapling has a lot of documentation, so we try to follow a guideline called the [Diátaxis documentation framework](https://diataxis.fr/). From 0b3509a295997c2361df77dd3a6142ae7f4d9b50 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 05:05:55 +0300 Subject: [PATCH 25/35] fix(shell): Fix a small typo --- scrapling/core/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 2b23b2b..82f9b7e 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -297,7 +297,7 @@ class CurlParser: _ = request_args.pop("json", None) try: - return getattr(self.__Fetcher, method)(**request_args) + return getattr(self.__fetcher, method)(**request_args) except Exception as e: # pragma: no cover log.error(f"Error calling Fetcher.{method}: {e}") return None From 1024ba6916b916b1cf55a2a73cd0411dc0c0bcf3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 16:30:59 +0300 Subject: [PATCH 26/35] tests: Update test to be up-to-date with current version of the code --- tests/cli/test_cli.py | 182 +++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 348c1ce..8acf28f 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -3,12 +3,23 @@ from click.testing import CliRunner from unittest.mock import patch, MagicMock import pytest_httpbin +from scrapling.parser import Selector from scrapling.cli import ( shell, mcp, get, post, put, delete, fetch, stealthy_fetch ) @pytest_httpbin.use_class_based_httpbin +def configure_selector_mock(): + """Helper function to create a properly configured Selector mock""" + mock_response = MagicMock(spec=Selector) + mock_response.body = "Test content" + mock_response.get_all_text.return_value = "Test content" + mock_response.css_first.return_value = mock_response + mock_response.css.return_value = [mock_response] + return mock_response + + class TestCLI: """Test CLI functionality""" @@ -45,136 +56,129 @@ class TestCLI: output_file = tmp_path / "output.md" with patch('scrapling.fetchers.Fetcher.get') as mock_get: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_response.status = 200 mock_get.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - get, - [html_url, str(output_file)] - ) - assert result.exit_code == 0 + result = runner.invoke( + get, + [html_url, str(output_file)] + ) + assert result.exit_code == 0 # Test with various options with patch('scrapling.fetchers.Fetcher.get') as mock_get: mock_get.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - get, - [ - html_url, - str(output_file), - '-H', 'User-Agent: Test', - '--cookies', 'session=abc123', - '--timeout', '60', - '--proxy', 'http://proxy:8080', - '-s', '.content', - '-p', 'page=1' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + get, + [ + html_url, + str(output_file), + '-H', 'User-Agent: Test', + '--cookies', 'session=abc123', + '--timeout', '60', + '--proxy', 'http://proxy:8080', + '-s', '.content', + '-p', 'page=1' + ] + ) + assert result.exit_code == 0 def test_extract_post_command(self, runner, tmp_path, html_url): """Test extract `post` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.post') as mock_post: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_post.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - post, - [ - html_url, - str(output_file), - '-d', 'key=value', - '-j', '{"data": "test"}' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + post, + [ + html_url, + str(output_file), + '-d', 'key=value', + '-j', '{"data": "test"}' + ] + ) + assert result.exit_code == 0 def test_extract_put_command(self, runner, tmp_path, html_url): """Test extract `put` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.put') as mock_put: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_put.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - put, - [ - html_url, - str(output_file), - '-d', 'key=value', - '-j', '{"data": "test"}' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + put, + [ + html_url, + str(output_file), + '-d', 'key=value', + '-j', '{"data": "test"}' + ] + ) + assert result.exit_code == 0 def test_extract_delete_command(self, runner, tmp_path, html_url): """Test extract `delete` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.delete') as mock_delete: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_delete.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - delete, - [ - html_url, - str(output_file) - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + delete, + [ + html_url, + str(output_file) + ] + ) + assert result.exit_code == 0 def test_extract_fetch_command(self, runner, tmp_path, html_url): """Test extract fetch command""" output_file = tmp_path / "output.txt" with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_fetch.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - fetch, - [ - html_url, - str(output_file), - '--headless', - '--stealth', - '--timeout', '60000' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + fetch, + [ + html_url, + str(output_file), + '--headless', + '--stealth', + '--timeout', '60000' + ] + ) + assert result.exit_code == 0 def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url): """Test extract fetch command""" output_file = tmp_path / "output.md" with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_fetch.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - stealthy_fetch, - [ - html_url, - str(output_file), - '--headless', - '--css-selector', 'body', - '--timeout', '60000' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + stealthy_fetch, + [ + html_url, + str(output_file), + '--headless', + '--css-selector', 'body', + '--timeout', '60000' + ] + ) + assert result.exit_code == 0 def test_invalid_arguments(self, runner, html_url): """Test invalid arguments handling""" @@ -182,12 +186,8 @@ class TestCLI: result = runner.invoke(get) assert result.exit_code != 0 - # Invalid output file extension - with patch('scrapling.cli.Convertor.write_content_to_file') as mock_write: - mock_write.side_effect = ValueError("Unknown file type") - - _ = runner.invoke( - get, - [html_url, 'output.invalid'] - ) - # Should handle the error gracefully + _ = runner.invoke( + get, + [html_url, 'output.invalid'] + ) + # Should handle the error gracefully From 68615b23472816a964f54a2f5ab8a6d72cb7367f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 16:31:38 +0300 Subject: [PATCH 27/35] tests: remove the cloudflare test from the sync version To not put pressure on the website --- tests/fetchers/sync/test_camoufox.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.py index 0207777..5a9209d 100644 --- a/tests/fetchers/sync/test_camoufox.py +++ b/tests/fetchers/sync/test_camoufox.py @@ -22,11 +22,6 @@ class TestStealthyFetcher: self.html_url = f"{httpbin.url}/html" self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response self.cookies_url = f"{httpbin.url}/cookies/set/test/value" - self.cloudflare_url = "https://nopecha.com/demo/cloudflare" # Interactive turnstile page - - def test_cloudflare_fetch(self, fetcher): - """Test if Cloudflare bypass is working""" - assert fetcher.fetch(self.cloudflare_url, solve_cloudflare=True).status == 200 def test_basic_fetch(self, fetcher): """Test doing a basic fetch request with multiple statuses""" From 67ca139ff9c5497aee57b9172eadda0804e39d61 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 20:20:29 +0300 Subject: [PATCH 28/35] fix: Fixes for multiple encoding issues (#80 & #81 ) --- scrapling/core/shell.py | 4 +- scrapling/engines/toolbelt/convertor.py | 155 +++++++++++++++--------- scrapling/engines/toolbelt/custom.py | 78 ------------ scrapling/parser.py | 8 +- 4 files changed, 101 insertions(+), 144 deletions(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 82f9b7e..8a38391 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -317,7 +317,7 @@ def show_page_in_browser(page: Selector): # pragma: no cover try: fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html") - with open(fd, "w", encoding="utf-8") as f: + with open(fd, "w", encoding=page.encoding) as f: f.write(page.body) open_in_browser(f"file://{fname}") @@ -556,7 +556,7 @@ class Convertor: elif not filename.endswith((".md", ".html", ".txt")): raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'") else: - with open(filename, "w", encoding="utf-8") as f: + with open(filename, "w", encoding=page.encoding) as f: extension = filename.split(".")[-1] f.write( "".join( diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 6b0ca41..05dce87 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -1,10 +1,15 @@ +from functools import lru_cache +from re import compile as re_compile + from curl_cffi.requests import Response as CurlResponse from playwright.sync_api import Page as SyncPage, Response as SyncResponse from playwright.async_api import Page as AsyncPage, Response as AsyncResponse from scrapling.core.utils import log -from scrapling.core._types import Dict, Optional from .custom import Response, StatusText +from scrapling.core._types import Dict, Optional + +__CHARSET_RE__ = re_compile(r"charset=([\w-]+)") class ResponseFactory: @@ -17,6 +22,18 @@ class ResponseFactory: response objects, and managing encoding, headers, cookies, and other attributes. """ + @classmethod + @lru_cache(maxsize=16) + def __extract_browser_encoding(cls, content_type: str | None) -> Optional[str]: + """Extract browser encoding from headers. + Ex: from header "content-type: text/html; charset=utf-8" -> "utf-8 + """ + if content_type: + # Because Playwright can't do that by themselves like all libraries for some reason :3 + match = __CHARSET_RE__.search(content_type) + return match.group(1) if match else None + return None + @classmethod def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]: """Process response history to build a list of `Response` objects""" @@ -30,18 +47,23 @@ class ResponseFactory: history.insert( 0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - content="", - status=current_response.status if current_response else 301, - reason=(current_response.status_text or StatusText.get(current_response.status)) - if current_response - else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies=tuple(), - headers=current_response.all_headers() if current_response else {}, - request_headers=current_request.all_headers(), - **parser_arguments, + **{ + "url": current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + "content": "", + "status": current_response.status if current_response else 301, + "reason": (current_response.status_text or StatusText.get(current_response.status)) + if current_response + else StatusText.get(301), + "encoding": cls.__extract_browser_encoding( + current_response.headers.get("content-type", "") + ) + or "utf-8", + "cookies": tuple(), + "headers": current_response.all_headers() if current_response else {}, + "request_headers": current_request.all_headers(), + **parser_arguments, + } ), ) except Exception as e: # pragma: no cover @@ -85,8 +107,9 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - # This will be parsed inside `Response` - encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding + encoding = ( + cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" + ) # default encoding # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) @@ -98,16 +121,18 @@ class ResponseFactory: page_content = "" return Response( - url=page.url, - content=page_content, - status=final_response.status, - reason=status_text, - encoding=encoding, - cookies=tuple(dict(cookie) for cookie in page.context.cookies()), - headers=first_response.all_headers(), - request_headers=first_response.request.all_headers(), - history=history, - **parser_arguments, + **{ + "url": page.url, + "content": page_content, + "status": final_response.status, + "reason": status_text, + "encoding": encoding, + "cookies": tuple(dict(cookie) for cookie in page.context.cookies()), + "headers": first_response.all_headers(), + "request_headers": first_response.request.all_headers(), + "history": history, + **parser_arguments, + } ) @classmethod @@ -125,18 +150,23 @@ class ResponseFactory: history.insert( 0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - content="", - status=current_response.status if current_response else 301, - reason=(current_response.status_text or StatusText.get(current_response.status)) - if current_response - else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies=tuple(), - headers=await current_response.all_headers() if current_response else {}, - request_headers=await current_request.all_headers(), - **parser_arguments, + **{ + "url": current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + "content": "", + "status": current_response.status if current_response else 301, + "reason": (current_response.status_text or StatusText.get(current_response.status)) + if current_response + else StatusText.get(301), + "encoding": cls.__extract_browser_encoding( + current_response.headers.get("content-type", "") + ) + or "utf-8", + "cookies": tuple(), + "headers": await current_response.all_headers() if current_response else {}, + "request_headers": await current_request.all_headers(), + **parser_arguments, + } ), ) except Exception as e: # pragma: no cover @@ -180,8 +210,9 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - # This will be parsed inside `Response` - encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding + encoding = ( + cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" + ) # default encoding # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) @@ -193,16 +224,18 @@ class ResponseFactory: page_content = "" return Response( - url=page.url, - content=page_content, - status=final_response.status, - reason=status_text, - encoding=encoding, - cookies=tuple(dict(cookie) for cookie in await page.context.cookies()), - headers=await first_response.all_headers(), - request_headers=await first_response.request.all_headers(), - history=history, - **parser_arguments, + **{ + "url": page.url, + "content": page_content, + "status": final_response.status, + "reason": status_text, + "encoding": encoding, + "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()), + "headers": await first_response.all_headers(), + "request_headers": await first_response.request.all_headers(), + "history": history, + **parser_arguments, + } ) @staticmethod @@ -214,15 +247,17 @@ class ResponseFactory: :return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ return Response( - url=response.url, - content=response.content if isinstance(response.content, bytes) else response.content.encode(), - status=response.status_code, - reason=response.reason, - encoding=response.encoding or "utf-8", - cookies=dict(response.cookies), - headers=dict(response.headers), - request_headers=dict(response.request.headers), - method=response.request.method, - history=response.history, # https://github.com/lexiforest/curl_cffi/issues/82 - **parser_arguments, + **{ + "url": response.url, + "content": response.content, + "status": response.status_code, + "reason": response.reason, + "encoding": response.encoding or "utf-8", + "cookies": dict(response.cookies), + "headers": dict(response.headers), + "request_headers": dict(response.request.headers), + "method": response.request.method, + "history": response.history, # https://github.com/lexiforest/curl_cffi/issues/82 + **parser_arguments, + } ) diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index f9effa2..4c4c5e7 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -17,81 +17,6 @@ from scrapling.core.custom_types import MappingProxyType from scrapling.parser import Selector, SQLiteStorageSystem -class ResponseEncoding: - __DEFAULT_ENCODING = "utf-8" - __ISO_8859_1_CONTENT_TYPES = { - "text/plain", - "text/html", - "text/css", - "text/javascript", - } - - @classmethod - @lru_cache(maxsize=128) - def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]: - """Parse content type and parameters from a content-type header value. - - Uses `email.message.Message` for robust header parsing according to RFC 2045. - - :param header_value: Raw content-type header string - :return: Tuple of (content_type, parameters_dict) - """ - # Create a Message object and set the Content-Type header then get the content type and parameters - msg = Message() - msg["content-type"] = header_value - - content_type = msg.get_content_type() - params = dict(msg.get_params(failobj=[])) - - # Remove the content-type from params if present somehow - params.pop("content-type", None) - - return content_type, params - - @classmethod - @lru_cache(maxsize=128) - def get_value(cls, content_type: Optional[str], text: Optional[str] = "test") -> str: - """Determine the appropriate character encoding from a content-type header. - - The encoding is determined by these rules in order: - 1. If no content-type is provided, use UTF-8 - 2. If charset parameter is present, use that encoding - 3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec - 4. If content-type is application/json, use UTF-8 per RFC 4627 - 5. Default to UTF-8 if nothing else matches - - :param content_type: Content-Type header value or None - :param text: A text to test the encoding on it - :return: String naming the character encoding - """ - if not content_type: - return cls.__DEFAULT_ENCODING - - try: - encoding = None - content_type, params = cls.__parse_content_type(content_type) - - # First check for explicit charset parameter - if "charset" in params: - encoding = params["charset"].strip("'\"") - - # Apply content-type specific rules - elif content_type in cls.__ISO_8859_1_CONTENT_TYPES: - encoding = "ISO-8859-1" - - elif content_type == "application/json": - encoding = cls.__DEFAULT_ENCODING - - if encoding: - _ = text.encode(encoding) # Validate encoding and validate it can encode the given text - return encoding - - return cls.__DEFAULT_ENCODING - - except (ValueError, LookupError, UnicodeEncodeError): - return cls.__DEFAULT_ENCODING - - class Response(Selector): """This class is returned by all engines as a way to unify response type between different libraries.""" @@ -116,9 +41,6 @@ class Response(Selector): self.headers = headers self.request_headers = request_headers self.history = history or [] - encoding = ResponseEncoding.get_value( - encoding, content.decode("utf-8") if isinstance(content, bytes) else content - ) super().__init__( content=content, url=adaptive_domain or url, diff --git a/scrapling/parser.py b/scrapling/parser.py index 5335a82..07576b0 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -74,7 +74,7 @@ class Selector(SelectorsGeneration): self, content: Optional[str | bytes] = None, url: Optional[str] = None, - encoding: str = "utf8", + encoding: str = "utf-8", huge_tree: bool = True, root: Optional[HtmlElement] = None, keep_comments: Optional[bool] = False, @@ -116,7 +116,7 @@ class Selector(SelectorsGeneration): if isinstance(content, str): body = content.strip().replace("\x00", "").encode(encoding) or b"" elif isinstance(content, bytes): - body = content.replace(b"\x00", b"").strip() + body = content.replace(b"\x00", b"") else: raise TypeError(f"content argument must be str or bytes, got {type(content)}") @@ -340,7 +340,7 @@ class Selector(SelectorsGeneration): @property def html_content(self) -> TextHandler: """Return the inner HTML code of the element""" - return TextHandler(tostring(self._root, encoding="unicode", method="html", with_tail=False)) + return TextHandler(tostring(self._root, encoding=self.encoding, method="html", with_tail=False)) body = html_content @@ -349,7 +349,7 @@ class Selector(SelectorsGeneration): return TextHandler( tostring( self._root, - encoding="unicode", + encoding=self.encoding, pretty_print=True, method="html", with_tail=False, From ce4fc31f2c30a322ba1bbd05bf43c3ba845f91b0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 20:21:45 +0300 Subject: [PATCH 29/35] feat: Make `.body` return the passed content as it is without any processing This makes it possible to download files and deal with non-HTML requests (ex: #81 ) --- scrapling/parser.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 07576b0..409e09d 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -132,8 +132,7 @@ class Selector(SelectorsGeneration): strip_cdata=(not keep_cdata), ) self._root = fromstring(body, parser=parser, base_url=url) - - self._raw_body = body.decode() + self._raw_body = content else: # All HTML types inherit from HtmlMixin so this to check for all at once @@ -342,7 +341,10 @@ class Selector(SelectorsGeneration): """Return the inner HTML code of the element""" return TextHandler(tostring(self._root, encoding=self.encoding, method="html", with_tail=False)) - body = html_content + @property + def body(self): + """Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests.""" + return self._raw_body def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" @@ -934,7 +936,7 @@ class Selector(SelectorsGeneration): # Operations on text functions def json(self) -> Dict: """Return JSON response if the response is jsonable otherwise throws error""" - if self._raw_body: + if self._raw_body and isinstance(self._raw_body, str): return TextHandler(self._raw_body).json() elif self.text: return self.text.json() From eae088a07b7e401a318271efb20cdcd6eae0b364 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 20:24:46 +0300 Subject: [PATCH 30/35] tests: Multiple changes to tests - Remove all tests for the old encoding logic - Stop using the `nopecha` test page to test Cloudflare solver - Remove useless tests like testing for infinite timeout - Fixes to make the code compatible with new changes --- tests/cli/test_cli.py | 1 + tests/fetchers/async/test_camoufox.py | 14 ++------------ tests/fetchers/async/test_dynamic.py | 6 ------ tests/fetchers/sync/test_camoufox.py | 4 ---- tests/fetchers/test_response_handling.py | 15 --------------- tests/fetchers/test_utils.py | 8 +------- tests/parser/test_parser_advanced.py | 2 +- 7 files changed, 5 insertions(+), 45 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 8acf28f..cae9fa0 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -14,6 +14,7 @@ def configure_selector_mock(): """Helper function to create a properly configured Selector mock""" mock_response = MagicMock(spec=Selector) mock_response.body = "Test content" + mock_response.encoding = "utf-8" mock_response.get_all_text.return_value = "Test content" mock_response.css_first.return_value = mock_response mock_response.css.return_value = [mock_response] diff --git a/tests/fetchers/async/test_camoufox.py b/tests/fetchers/async/test_camoufox.py index 6a0700b..ffe6eac 100644 --- a/tests/fetchers/async/test_camoufox.py +++ b/tests/fetchers/async/test_camoufox.py @@ -1,3 +1,4 @@ +from playwright._impl._errors import TimeoutError import pytest import pytest_httpbin @@ -23,14 +24,9 @@ class TestStealthyFetcher: "basic_url": f"{url}/get", "html_url": f"{url}/html", "delayed_url": f"{url}/delay/10", # 10 Seconds delay response - "cookies_url": f"{url}/cookies/set/test/value", - "cloudflare_url": "https://nopecha.com/demo/cloudflare", # Interactive turnstile page + "cookies_url": f"{url}/cookies/set/test/value" } - async def test_cloudflare_fetch(self, fetcher, urls): - """Test if Cloudflare bypass is working""" - assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200 - async def test_basic_fetch(self, fetcher, urls): """Test doing a basic fetch request with multiple statuses""" assert (await fetcher.async_fetch(urls["status_200"])).status == 200 @@ -86,9 +82,3 @@ class TestStealthyFetcher: **kwargs ) assert response.status == 200 - - async def test_infinite_timeout(self, fetcher, urls): - """Test if infinite timeout breaks the code or not""" - assert ( - await fetcher.async_fetch(urls["delayed_url"], timeout=0) - ).status == 200 diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py index 0d171ce..04106fa 100644 --- a/tests/fetchers/async/test_dynamic.py +++ b/tests/fetchers/async/test_dynamic.py @@ -90,9 +90,3 @@ class TestDynamicFetcherAsync: with pytest.raises(Exception): await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah") - - @pytest.mark.asyncio - async def test_infinite_timeout(self, fetcher, urls): - """Test if infinite timeout breaks the code or not""" - response = await fetcher.async_fetch(urls["delayed_url"], timeout=0) - assert response.status == 200 diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.py index 5a9209d..83c5f6c 100644 --- a/tests/fetchers/sync/test_camoufox.py +++ b/tests/fetchers/sync/test_camoufox.py @@ -77,7 +77,3 @@ class TestStealthyFetcher: **kwargs ) assert response.status == 200 - - def test_infinite_timeout(self, fetcher): - """Test if infinite timeout breaks the code or not""" - assert fetcher.fetch(self.delayed_url, timeout=0).status == 200 diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py index 4db96bf..1327db8 100644 --- a/tests/fetchers/test_response_handling.py +++ b/tests/fetchers/test_response_handling.py @@ -1,7 +1,6 @@ from unittest.mock import Mock from scrapling.parser import Selector -from scrapling.engines.toolbelt.custom import ResponseEncoding from scrapling.engines.toolbelt.convertor import ResponseFactory, Response @@ -32,20 +31,6 @@ class TestResponseFactory: assert response.url == "https://example.com" assert isinstance(response, Response) - def test_response_encoding_edge_cases(self): - """Test response encoding handling""" - # Test various content types - test_cases = [ - (None, "utf-8"), - ("", "utf-8"), - ("text/html; charset=invalid", "utf-8"), - ("application/octet-stream", "utf-8"), - ] - - for content_type, expected in test_cases: - encoding = ResponseEncoding.get_value(content_type) - assert encoding == expected - def test_response_history_processing(self): """Test processing response history""" # Mock responses with redirects diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 4a1563c..0028902 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,7 +1,7 @@ import pytest from pathlib import Path -from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response +from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, js_bypass_path @@ -131,12 +131,6 @@ def status_map(): } -def test_parsing_content_type(content_type_map): - """Test if parsing different types of 'content-type' returns the expected result""" - for header_value, expected_encoding in content_type_map.items(): - assert ResponseEncoding.get_value(header_value) == expected_encoding - - def test_parsing_response_status(status_map): """Test if using different http responses' status codes returns the expected result""" for status_code, expected_status_text in status_map.items(): diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index 71552f9..3ac81cf 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -99,7 +99,7 @@ class TestAdvancedSelectors: keep_comments=False, keep_cdata=False ) - content = page.body + content = page.html_content assert "Comment" not in content def test_advanced_xpath_variables(self, complex_html): From fdac239fcc4a83f6a97a11df5b43657917e67190 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 20:32:46 +0300 Subject: [PATCH 31/35] build: Move click library to be part of the fetchers extra --- pyproject.toml | 2 +- scrapling/cli.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c708f3..53f5a4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,13 +58,13 @@ classifiers = [ dependencies = [ "lxml>=6.0.1", "cssselect>=1.3.0", - "click>=8.2.1", "orjson>=3.11.3", "tldextract>=5.3.0", ] [project.optional-dependencies] fetchers = [ + "click>=8.2.1", "curl_cffi>=0.13.0", "playwright>=1.52.0", "rebrowser-playwright>=1.52.0", diff --git a/scrapling/cli.py b/scrapling/cli.py index 3e052ac..0b68721 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -7,7 +7,13 @@ from scrapling.core.utils import log, _CookieParser, _ParseHeaders from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable from orjson import loads as json_loads, JSONDecodeError -from click import command, option, Choice, group, argument + +try: + from click import command, option, Choice, group, argument +except ImportError: + raise ImportError( + "You need to install scrapling with any of the extras to enable Shell commands. See: https://scrapling.readthedocs.io/en/latest/#installation" + ) __OUTPUT_FILE_HELP__ = "The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively." __PACKAGE_DIR__ = Path(__file__).parent From d259b85c5b9b9d5ee2a65d74c4718efc48cb887e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 21:26:28 +0300 Subject: [PATCH 32/35] docs: Updating benchmarks with the latest version of all libraries --- README.md | 24 ++++++++++++------------ docs/benchmarks.md | 22 +++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 88ba6f8..28e549d 100644 --- a/README.md +++ b/README.md @@ -236,20 +236,20 @@ scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas. ## Performance Benchmarks -Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations! +Scrapling isn't just powerful—it's also blazing fast, and the updates since version 0.3 deliver exceptional performance improvements across all operations! ### Text Extraction Speed Test (5000 nested elements) | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.88 | 1.0x | -| 2 | Parsel/Scrapy | 1.96 | 1.043x | -| 3 | Raw Lxml | 2.32 | 1.234x | -| 4 | PyQuery | 20.2 | ~11x | -| 5 | Selectolax | 85.2 | ~45x | -| 6 | MechanicalSoup | 1305.84 | ~695x | -| 7 | BS4 with Lxml | 1307.92 | ~696x | -| 8 | BS4 with html5lib | 3336.28 | ~1775x | +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | ### Element Similarity & Text Search Performance @@ -257,8 +257,8 @@ Scrapling's adaptive element finding capabilities significantly outperform alter | Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 2.02 | 1.0x | -| AutoScraper | 10.26 | 5.08x | +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | > All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology. @@ -326,4 +326,4 @@ This project includes code adapted from: - [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches) for stealth improvements --- -
Designed & crafted with ❤️ by Karim Shoair.

+
Designed & crafted with ❤️ by Karim Shoair.

\ No newline at end of file diff --git a/docs/benchmarks.md b/docs/benchmarks.md index ceb35e4..4e207ad 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,6 +1,6 @@ # Performance Benchmarks -Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations! +Scrapling isn't just powerful—it's also blazing fast, and the updates since version 0.3 deliver exceptional performance improvements across all operations! ## Benchmark Results @@ -8,14 +8,14 @@ Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.88 | 1.0x | -| 2 | Parsel/Scrapy | 1.96 | 1.043x | -| 3 | Raw Lxml | 2.32 | 1.234x | -| 4 | PyQuery | 20.2 | ~11x | -| 5 | Selectolax | 85.2 | ~45x | -| 6 | MechanicalSoup | 1305.84 | ~695x | -| 7 | BS4 with Lxml | 1307.92 | ~696x | -| 8 | BS4 with html5lib | 3336.28 | ~1775x | +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | ### Element Similarity & Text Search Performance @@ -23,5 +23,5 @@ Scrapling's adaptive element finding capabilities significantly outperform alter | Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 2.02 | 1.0x | -| AutoScraper | 10.26 | 5.08x | +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | From c7b9d45232e0d128452386800cfe539feb98a82b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 14 Sep 2025 21:35:58 +0300 Subject: [PATCH 33/35] ops: Make tests workflow doesn't run when the changes isn't code related --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 88737fa..10a6740 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,15 @@ on: branches: - main - dev + paths-ignore: + - '*.md' + - '**/*.md' + - 'docs/*' + - 'images/*' + - '.github/*' + - '*.yml' + - '*.yaml' + - 'ruff.toml' concurrency: group: ${{github.workflow}}-${{ github.ref }} From f848e6b7f2e1adc24340c650e0bbc1c1335c716c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 15 Sep 2025 00:19:19 +0300 Subject: [PATCH 34/35] fix: Fixes to handle the new extra deps group --- scrapling/cli.py | 6 +++--- scrapling/core/ai.py | 2 +- scrapling/engines/static.py | 8 ++------ scrapling/engines/toolbelt/__init__.py | 12 +----------- scrapling/engines/toolbelt/custom.py | 1 - scrapling/fetchers.py | 2 +- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 0b68721..48e99bf 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -10,10 +10,10 @@ from orjson import loads as json_loads, JSONDecodeError try: from click import command, option, Choice, group, argument -except ImportError: - raise ImportError( +except (ImportError, ModuleNotFoundError) as e: + raise ModuleNotFoundError( "You need to install scrapling with any of the extras to enable Shell commands. See: https://scrapling.readthedocs.io/en/latest/#installation" - ) + ) from e __OUTPUT_FILE_HELP__ = "The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively." __PACKAGE_DIR__ = Path(__file__).parent diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 7777c21..ae517fa 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -4,7 +4,7 @@ from mcp.server.fastmcp import FastMCP from pydantic import BaseModel, Field from scrapling.core.shell import Convertor -from scrapling.engines.toolbelt import Response as _ScraplingResponse +from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse from scrapling.fetchers import ( Fetcher, FetcherSession, diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 35271b8..3f6cb79 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -24,13 +24,9 @@ from scrapling.core._types import ( Any, ) -from .toolbelt import ( - Response, - generate_headers, - __default_useragent__, -) +from .toolbelt.custom import Response from .toolbelt.convertor import ResponseFactory -from .toolbelt.fingerprints import generate_convincing_referer +from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ _UNSET = object() diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index 04f43b1..8b13789 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -1,11 +1 @@ -from .custom import ( - BaseFetcher, - Response, - StatusText, - get_variable_name, -) -from .fingerprints import ( - generate_headers, - get_os_name, - __default_useragent__, -) + diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 4c4c5e7..774eec7 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -3,7 +3,6 @@ Functions related to custom types or type checking """ from functools import lru_cache -from email.message import Message from scrapling.core.utils import log from scrapling.core._types import ( diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 323293d..fbb2b28 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -17,7 +17,7 @@ from scrapling.engines._browsers import ( AsyncDynamicSession, AsyncStealthySession, ) -from scrapling.engines.toolbelt import BaseFetcher, Response +from scrapling.engines.toolbelt.custom import BaseFetcher, Response __FetcherClientInstance__ = _FetcherClient() __AsyncFetcherClientInstance__ = _AsyncFetcherClient() From 3bf06a8521f8cdd5b7b32bda1ae9e50eda920b14 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 15 Sep 2025 01:39:19 +0300 Subject: [PATCH 35/35] docs: clarifying updates --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28e549d..f344704 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies. ### Optional Dependencies diff --git a/docs/index.md b/docs/index.md index ea7a2de..51d490d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -114,7 +114,7 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies. ### Optional Dependencies