From 7ba333c3c9f6678a0a15787fb0cbc2136904e842 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:49:45 +0300 Subject: [PATCH 01/12] perf(session): improve page management and speed up execution Close pages immediately after tasks finish to reduce resource usage and improve overall performance. Update corresponding test files to reflect the new page lifecycle management. --- scrapling/engines/_browsers/_base.py | 10 ------ scrapling/engines/_browsers/_camoufox.py | 10 +++--- scrapling/engines/_browsers/_controllers.py | 11 +++--- scrapling/engines/_browsers/_page.py | 35 ------------------- tests/fetchers/async/test_camoufox_session.py | 12 ++++--- tests/fetchers/async/test_dynamic_session.py | 14 ++++---- tests/fetchers/test_pages.py | 18 ---------- 7 files changed, 27 insertions(+), 83 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 8e94559..6445e00 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -44,16 +44,11 @@ class SyncSession: ) -> PageInfo: # pragma: no cover """Get a new page to use""" - # Close all finished pages to ensure clean state - self.page_pool.close_all_finished_pages() - # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() while time() - start_time < self._max_wait_for_page: - # Wait for any pages to finish, then clean them up sleep(0.05) - self.page_pool.close_all_finished_pages() if self.page_pool.pages_count < self.max_pages: break else: @@ -105,16 +100,11 @@ class AsyncSession(SyncSession): ) -> PageInfo: # pragma: no cover """Get a new page to use""" async with self._lock: - # Close all finished pages to ensure clean state - await self.page_pool.aclose_all_finished_pages() - # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() while time() - start_time < self._max_wait_for_page: - # Wait for any pages to finish, then clean them up await asyncio_sleep(0.05) - await self.page_pool.aclose_all_finished_pages() if self.page_pool.pages_count < self.max_pages: break else: diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index a4ca6f0..18ec633 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -381,8 +381,9 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + page_info.page.close() + self.page_pool.pages.remove(page_info) return response @@ -701,8 +702,9 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + await page_info.page.close() + self.page_pool.pages.remove(page_info) return response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 70b3771..722d7c2 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -305,8 +305,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + page_info.page.close() + self.page_pool.pages.remove(page_info) return response @@ -554,9 +555,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() - + # Close the page, to free up resources + await page_info.page.close() + self.page_pool.pages.remove(page_info) return response except Exception as e: # pragma: no cover diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index ffa4b22..8c80944 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -23,11 +23,6 @@ class PageInfo: self.state = "busy" self.url = url - def mark_finished(self): - """Mark the page as finished for new requests""" - self.state = "finished" - self.url = "" - def mark_error(self): """Mark the page as having an error""" self.state = "error" @@ -83,33 +78,3 @@ class PagePool: """Remove pages in error state""" with self._lock: self.pages = [p for p in self.pages if p.state != "error"] - - def close_all_finished_pages(self): - """Close all pages in finished state and remove them from the pool""" - with self._lock: - pages_to_remove = [] - for page_info in self.pages: - if page_info.state == "finished": - try: - page_info.page.close() - except Exception: - pass - pages_to_remove.append(page_info) - - for page_info in pages_to_remove: - self.pages.remove(page_info) - - async def aclose_all_finished_pages(self): - """Async version: Close all pages in finished state and remove them from the pool""" - with self._lock: - pages_to_remove = [] - for page_info in self.pages: - if page_info.state == "finished": - try: - await page_info.page.close() - except Exception: - pass - pages_to_remove.append(page_info) - - for page_info in pages_to_remove: - self.pages.remove(page_info) diff --git a/tests/fetchers/async/test_camoufox_session.py b/tests/fetchers/async/test_camoufox_session.py index 2971488..af9b32b 100644 --- a/tests/fetchers/async/test_camoufox_session.py +++ b/tests/fetchers/async/test_camoufox_session.py @@ -54,16 +54,18 @@ class TestAsyncStealthySession: """Test page pool creation and reuse""" async with AsyncStealthySession() as session: # The first request creates a page - _ = await session.fetch(urls["basic"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["basic"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # The second request should reuse the page - _ = await session.fetch(urls["html"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["html"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # Check pool stats stats = session.get_pool_stats() - assert stats["total_pages"] == 1 + assert stats["total_pages"] == 0 assert stats["max_pages"] == 1 async def test_stealthy_session_with_options(self, urls): diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py index d7a4ea9..e6f0b5a 100644 --- a/tests/fetchers/async/test_dynamic_session.py +++ b/tests/fetchers/async/test_dynamic_session.py @@ -53,16 +53,18 @@ class TestAsyncDynamicSession: """Test page pool creation and reuse""" async with AsyncDynamicSession() as session: # The first request creates a page - _ = await session.fetch(urls["basic"]) - assert session.page_pool.pages_count == 1 - + response = await session.fetch(urls["basic"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 + # The second request should reuse the page - _ = await session.fetch(urls["html"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["html"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # Check pool stats stats = session.get_pool_stats() - assert stats["total_pages"] == 1 + assert stats["total_pages"] == 0 assert stats["max_pages"] == 1 async def test_dynamic_session_with_options(self, urls): diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index e3ba3bc..69dc4ba 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -24,10 +24,6 @@ class TestPageInfo: assert page_info.state == "busy" assert page_info.url == "https://example.com" - page_info.mark_finished() - assert page_info.state == "finished" - assert page_info.url == "" - page_info.mark_error() assert page_info.state == "error" @@ -88,21 +84,7 @@ class TestPagePool: with pytest.raises(RuntimeError): pool.add_page(Mock()) - def test_get_ready_page(self): - """Test getting ready page""" - pool = PagePool(max_pages=3) - # Add pages - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - - # Mark them as finished - page1.mark_finished() - page2.mark_finished() - - # test - pool.close_all_finished_pages() - assert pool.pages_count == 0 def test_cleanup_error_pages(self): """Test cleaning up error pages""" From b197f8a5f99e74967e62cf986f80d26a15a02d11 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Sep 2025 13:56:37 +0300 Subject: [PATCH 02/12] fix(parser): Improve selectors `re` function --- scrapling/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 4fb510c..6260988 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1259,7 +1259,7 @@ class Selectors(List[Selector]): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it """ - results = [n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self] + results = [n.re(regex, replace_entities, clean_match, case_sensitive) for n in self] return TextHandlers(flatten(results)) def re_first( From 504ecdf7dc7376793984bcbb9037577a1c0ec1a9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 04:20:16 +0300 Subject: [PATCH 03/12] fix(shell): Fix parsing curl `--data-raw` --- scrapling/core/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 58ec258..1bd67ea 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -201,7 +201,7 @@ class CurlParser: data_payload = parsed_args.data_binary # Fallback to string elif parsed_args.data_raw is not None: - data_payload = parsed_args.data_raw + data_payload = parsed_args.data_raw.lstrip("$") elif parsed_args.data is not None: data_payload = parsed_args.data From bf5fe6d45189df443b2d2675f7475887bef785d3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 04:21:15 +0300 Subject: [PATCH 04/12] fix(shell): Fix `view` command edge case --- scrapling/core/shell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 1bd67ea..4cca171 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -317,8 +317,8 @@ def show_page_in_browser(page: Selector): # pragma: no cover try: fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html") - with open(fd, "wb") as f: - f.write(page.body) + with open(fd, "w", encoding=page.encoding) as f: + f.write(page.html_content) open_in_browser(f"file://{fname}") except IOError as e: From a9d05cc0ef495b329618be2824554e69856e5b27 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 04:49:30 +0300 Subject: [PATCH 05/12] style: Removing dead code/docstrings and correcting type hints --- scrapling/core/custom_types.py | 4 ++-- scrapling/engines/_browsers/_page.py | 8 +------- scrapling/engines/static.py | 6 ++---- scrapling/engines/toolbelt/navigation.py | 2 +- scrapling/parser.py | 4 ++-- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index eb7fa34..e9ece7d 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -145,7 +145,7 @@ class TextHandler(str): clean_match: bool = False, case_sensitive: bool = True, check_match: Literal[False] = False, - ) -> "TextHandlers[TextHandler]": ... + ) -> "TextHandlers": ... def re( self, @@ -241,7 +241,7 @@ class TextHandlers(List[TextHandler]): replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, - ) -> "TextHandlers[TextHandler]": + ) -> "TextHandlers": """Call the ``.re()`` method for each element in this list and return their results flattened as TextHandlers. diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 8c80944..821fbbb 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,7 +6,7 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in +PageState = Literal["ready", "busy", "error"] # States that a page can be in @dataclass @@ -62,12 +62,6 @@ class PagePool: """Get the total number of pages""" return len(self.pages) - @property - def finished_count(self) -> int: - """Get the number of finished pages""" - with self._lock: - return sum(1 for p in self.pages if p.state == "finished") - @property def busy_count(self) -> int: """Get the number of busy pages""" diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 3f6cb79..6e5c0de 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -94,8 +94,8 @@ class FetcherSession: self.default_http3 = http3 self.selector_config = selector_config or {} - self._curl_session: Optional[CurlSession] = None - self._async_curl_session: Optional[AsyncCurlSession] = None + self._curl_session: Optional[CurlSession] | bool = None + self._async_curl_session: Optional[AsyncCurlSession] | bool = None def _merge_request_args(self, **kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" @@ -239,7 +239,6 @@ class FetcherSession: Perform an HTTP request using the configured session. :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param url: Target URL for the request. :param request_args: Arguments to be passed to the session's `request()` method. :param max_retries: Maximum number of retries for the request. :param retry_delay: Number of seconds to wait between retries. @@ -280,7 +279,6 @@ class FetcherSession: Perform an HTTP request using the configured session. :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param url: Target URL for the request. :param request_args: Arguments to be passed to the session's `request()` method. :param max_retries: Maximum number of retries for the request. :param retry_delay: Number of seconds to wait between retries. diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index f2f445c..ea5991b 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -4,7 +4,7 @@ Functions related to files and URLs from pathlib import Path from functools import lru_cache -from urllib.parse import urlencode, urlparse +from urllib.parse import urlparse from playwright.async_api import Route as async_Route from msgspec import Struct, structs, convert, ValidationError diff --git a/scrapling/parser.py b/scrapling/parser.py index 6260988..f0b9e54 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -239,7 +239,7 @@ class Selector(SelectorsGeneration): ) def __handle_element( - self, element: HtmlElement | _ElementUnicodeResult + self, element: Optional[HtmlElement | _ElementUnicodeResult] ) -> Optional[Union[TextHandler, "Selector"]]: """Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible""" if element is None: @@ -345,7 +345,7 @@ class Selector(SelectorsGeneration): return TextHandler(content) @property - def body(self): + def body(self) -> str | bytes: """Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests.""" return self._raw_body From 2d704b2a8b9d3f4f25e72c155e7c0ed18331913d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 04:50:15 +0300 Subject: [PATCH 06/12] fix(shell): Fixing a bug with content converting --- scrapling/cli.py | 8 ++++---- scrapling/core/shell.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 48e99bf..12b96c6 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -32,8 +32,8 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any try: return json_loads(json_string) - except JSONDecodeError as e: # pragma: no cover - raise ValueError(f"Invalid JSON data '{json_string}': {e}") + except JSONDecodeError as err: # pragma: no cover + raise ValueError(f"Invalid JSON data '{json_string}': {err}") def __Request_and_Save( @@ -65,8 +65,8 @@ def __ParseExtractArguments( for key, value in _CookieParser(cookies): try: parsed_cookies[key] = value - except Exception as e: - raise ValueError(f"Could not parse cookies '{cookies}': {e}") + except Exception as err: + raise ValueError(f"Could not parse cookies '{cookies}': {err}") parsed_json = __ParseJSONData(json) parsed_params = {} diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 4cca171..f9f790f 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -545,7 +545,7 @@ class Convertor: for page in pages: match extraction_type: case "markdown": - yield cls._convert_to_markdown(page.body) + yield cls._convert_to_markdown(page.html_content) case "html": yield page.body case "text": From 386da62ec9b76671ec37bb2c9d51c693fb0d1da2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 23:17:54 +0300 Subject: [PATCH 07/12] tests: fix CLI tests according to last changes --- tests/cli/test_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index cae9fa0..1f98bfb 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -14,6 +14,7 @@ def configure_selector_mock(): """Helper function to create a properly configured Selector mock""" mock_response = MagicMock(spec=Selector) mock_response.body = "Test content" + mock_response.html_content = "Test content" mock_response.encoding = "utf-8" mock_response.get_all_text.return_value = "Test content" mock_response.css_first.return_value = mock_response From 67db865aee48af6375a344dd1ef7b26626ed0b19 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 20 Sep 2025 00:23:29 +0300 Subject: [PATCH 08/12] feat(DynamicFetcher): Replace `rebrowser` with `patchright` --- .github/workflows/tests.yml | 2 +- README.md | 5 ++--- docs/fetching/dynamic.md | 2 +- pyproject.toml | 4 ++-- scrapling/engines/_browsers/_controllers.py | 16 ++++------------ tox.ini | 4 ++-- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10a6740..4a62cb1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: - name: Install all browsers dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox + python3 -m pip install playwright>=1.55.0 patchright>=1.55.0 camoufox - name: Retrieve Playwright browsers from cache if any id: playwright-cache diff --git a/README.md b/README.md index 6c63c63..d5171c7 100644 --- a/README.md +++ b/README.md @@ -322,10 +322,9 @@ This project includes code adapted from: ## Thanks and References - [Daijro](https://github.com/daijro)'s brilliant work on [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox) -- [Vinyzu](https://github.com/Vinyzu)'s work on [Botright](https://github.com/Vinyzu/Botright) +- [Vinyzu](https://github.com/Vinyzu)'s brilliant work on [Botright](https://github.com/Vinyzu/Botright) and [PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright) - [brotector](https://github.com/kaliiiiiiiiii/brotector) for browser detection bypass techniques -- [fakebrowser](https://github.com/kkoooqq/fakebrowser) for fingerprinting research -- [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches) for stealth improvements +- [fakebrowser](https://github.com/kkoooqq/fakebrowser) and [BotBrowser](https://github.com/botswin/BotBrowser) for fingerprinting research ---
Designed & crafted with ❤️ by Karim Shoair.

\ No newline at end of file diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 11b5e41..95f6a87 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -35,7 +35,7 @@ It's the same as the vanilla Playwright option, but it provides a simple stealth Some of the things this fetcher's stealth mode does include: - * Patching the CDP runtime fingerprint. + * Patching the CDP runtime fingerprint through using PatchRight. * Mimics some of the real browsers' properties by injecting several JS files and using custom options. * Custom flags are used on launch to hide Playwright even more and make it faster. * Generates real browser headers of the same type and user OS, then appends them to the request's headers. diff --git a/pyproject.toml b/pyproject.toml index 53f5a4c..ba3b868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,8 @@ dependencies = [ fetchers = [ "click>=8.2.1", "curl_cffi>=0.13.0", - "playwright>=1.52.0", - "rebrowser-playwright>=1.52.0", + "playwright>=1.55.0", + "patchright>=1.55.2", "camoufox>=0.4.11", "geoip2>=5.1.0", "msgspec>=0.19.0", diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 722d7c2..19362e7 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -11,10 +11,8 @@ from playwright.async_api import ( Playwright as AsyncPlaywright, Locator as AsyncLocator, ) -from rebrowser_playwright.sync_api import sync_playwright as sync_rebrowser_playwright -from rebrowser_playwright.async_api import ( - async_playwright as async_rebrowser_playwright, -) +from patchright.sync_api import sync_playwright as sync_patchright +from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin @@ -154,10 +152,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): def __create__(self): """Create a browser for this instance and context.""" - sync_context = sync_rebrowser_playwright - if not self.stealth or self.real_chrome: - # Because rebrowser_playwright doesn't play well with real browsers - sync_context = sync_playwright + sync_context = sync_patchright if self.stealth else sync_playwright self.playwright: Playwright = sync_context().start() @@ -403,10 +398,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): async def __create__(self): """Create a browser for this instance and context.""" - async_context = async_rebrowser_playwright - if not self.stealth or self.real_chrome: - # Because rebrowser_playwright doesn't play well with real browsers - async_context = async_playwright + async_context = async_patchright if self.stealth else async_playwright self.playwright: AsyncPlaywright = await async_context().start() diff --git a/tox.ini b/tox.ini index c20798f..758939f 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,8 @@ envlist = pre-commit,py{310,311,312,313} usedevelop = True changedir = tests deps = - playwright==1.52.0 - rebrowser-playwright==1.52.0 + playwright>=1.55.0 + patchright>=1.55.0 camoufox -r{toxinidir}/tests/requirements.txt extras = ai,shell From 67aa1a2ae7e147bca3b19d5defe62ae552b48430 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 20 Sep 2025 00:23:55 +0300 Subject: [PATCH 09/12] tests: Add a test for using `stealth` and `real_chrome` arguments together --- tests/fetchers/async/test_dynamic.py | 1 + tests/fetchers/sync/test_dynamic.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py index 04106fa..5174c9f 100644 --- a/tests/fetchers/async/test_dynamic.py +++ b/tests/fetchers/async/test_dynamic.py @@ -56,6 +56,7 @@ class TestDynamicFetcherAsync: {"disable_webgl": True, "hide_canvas": False}, {"disable_webgl": False, "hide_canvas": True, "disable_resources": True}, {"stealth": True}, # causes issues with GitHub Actions + {"stealth": True, "real_chrome": True}, # causes issues with GitHub Actions {"wait_selector": "h1", "wait_selector_state": "attached"}, {"wait_selector": "h1", "wait_selector_state": "visible"}, { diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_dynamic.py index 4804acc..a140076 100644 --- a/tests/fetchers/sync/test_dynamic.py +++ b/tests/fetchers/sync/test_dynamic.py @@ -54,6 +54,7 @@ class TestDynamicFetcher: {"disable_webgl": True, "hide_canvas": False}, {"disable_webgl": False, "hide_canvas": True, "disable_resources": True}, {"stealth": True}, # causes issues with GitHub Actions + {"stealth": True, "real_chrome": True}, # causes issues with GitHub Actions {"wait_selector": "h1", "wait_selector_state": "attached"}, {"wait_selector": "h1", "wait_selector_state": "visible"}, { From 5164b801ec654b3e111d4e359aac9f956c8b77a9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 20 Sep 2025 00:33:07 +0300 Subject: [PATCH 10/12] build: pump version up and update deps --- pyproject.toml | 4 ++-- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ba3b868..1175b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ dependencies = [ [project.optional-dependencies] fetchers = [ - "click>=8.2.1", + "click>=8.3.0", "curl_cffi>=0.13.0", "playwright>=1.55.0", "patchright>=1.55.2", @@ -73,7 +73,7 @@ fetchers = [ "msgspec>=0.19.0", ] ai = [ - "mcp>=1.14.0", + "mcp>=1.14.1", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 99e72b7..d797639 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.4" +__version__ = "0.3.5" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index 5e52c3f..629514a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.4 +version = 0.3.5 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From 970ad40404c2ae8cb1d91236095b7321f27658cd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 20 Sep 2025 04:31:57 +0300 Subject: [PATCH 11/12] perf: Speed up sync fetches by ignoring number of pages checking --- scrapling/engines/_browsers/_base.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6445e00..703a915 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -1,4 +1,4 @@ -from time import time, sleep +from time import time from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons @@ -44,18 +44,7 @@ class SyncSession: ) -> PageInfo: # pragma: no cover """Get a new page to use""" - # If we're at max capacity after cleanup, wait for busy pages to finish - if self.page_pool.pages_count >= self.max_pages: - start_time = time() - while time() - start_time < self._max_wait_for_page: - sleep(0.05) - if self.page_pool.pages_count < self.max_pages: - break - else: - raise TimeoutError( - f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" - ) - + # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc. page = self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) From 82b09620b28f743265430ba43c351244b2de40c2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 20 Sep 2025 06:20:23 +0300 Subject: [PATCH 12/12] perf: Improving all browser-based fetches by ~15% --- scrapling/engines/_browsers/_base.py | 5 - scrapling/engines/_browsers/_camoufox.py | 67 ++++--- scrapling/engines/_browsers/_controllers.py | 64 +++---- scrapling/engines/_browsers/_validators.py | 197 +++++++++++++------- 4 files changed, 195 insertions(+), 138 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 703a915..69681ee 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -60,11 +60,6 @@ class SyncSession: return self.page_pool.add_page(page) - @staticmethod - def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any: - """Get value with request-level priority over session-level""" - return request_value if request_value is not sentinel_value else session_value - def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" return { diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 18ec633..5f0e795 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -16,7 +16,7 @@ from playwright.async_api import ( ) from playwright._impl._errors import Error as PlaywrightError -from ._validators import validate, CamoufoxConfig +from ._validators import validate_fetch as _validate from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( @@ -297,23 +297,22 @@ class StealthySession(StealthySessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - # Validate all resolved parameters - params = validate( - dict( - google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), - timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), - wait=self._get_with_precedence(wait, self.wait, _UNSET), - page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), - extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), - disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), - wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), - wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), - network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), - load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), - solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), - selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), - ), - CamoufoxConfig, + params = _validate( + [ + ("google_search", google_search, self.google_search), + ("timeout", timeout, self.timeout), + ("wait", wait, self.wait), + ("page_action", page_action, self.page_action), + ("extra_headers", extra_headers, self.extra_headers), + ("disable_resources", disable_resources, self.disable_resources), + ("wait_selector", wait_selector, self.wait_selector), + ("wait_selector_state", wait_selector_state, self.wait_selector_state), + ("network_idle", network_idle, self.network_idle), + ("load_dom", load_dom, self.load_dom), + ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), + ("selector_config", selector_config, self.selector_config), + ], + _UNSET, ) if self._closed: # pragma: no cover @@ -617,22 +616,22 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = validate( - dict( - google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), - timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), - wait=self._get_with_precedence(wait, self.wait, _UNSET), - page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), - extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), - disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), - wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), - wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), - network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), - load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), - solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), - selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), - ), - CamoufoxConfig, + params = _validate( + [ + ("google_search", google_search, self.google_search), + ("timeout", timeout, self.timeout), + ("wait", wait, self.wait), + ("page_action", page_action, self.page_action), + ("extra_headers", extra_headers, self.extra_headers), + ("disable_resources", disable_resources, self.disable_resources), + ("wait_selector", wait_selector, self.wait_selector), + ("wait_selector_state", wait_selector_state, self.wait_selector_state), + ("network_idle", network_idle, self.network_idle), + ("load_dom", load_dom, self.load_dom), + ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), + ("selector_config", selector_config, self.selector_config), + ], + _UNSET, ) if self._closed: # pragma: no cover diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 19362e7..b4b6276 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -16,7 +16,7 @@ from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin -from ._validators import validate, PlaywrightConfig +from ._validators import validate_fetch as _validate from scrapling.core._types import ( Dict, List, @@ -224,22 +224,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - # Validate all resolved parameters - params = validate( - dict( - google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), - timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), - wait=self._get_with_precedence(wait, self.wait, _UNSET), - page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), - extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), - disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), - wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), - wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), - network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), - load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), - selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), - ), - PlaywrightConfig, + params = _validate( + [ + ("google_search", google_search, self.google_search), + ("timeout", timeout, self.timeout), + ("wait", wait, self.wait), + ("page_action", page_action, self.page_action), + ("extra_headers", extra_headers, self.extra_headers), + ("disable_resources", disable_resources, self.disable_resources), + ("wait_selector", wait_selector, self.wait_selector), + ("wait_selector_state", wait_selector_state, self.wait_selector_state), + ("network_idle", network_idle, self.network_idle), + ("load_dom", load_dom, self.load_dom), + ("selector_config", selector_config, self.selector_config), + ], + _UNSET, ) if self._closed: # pragma: no cover @@ -471,22 +470,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - # Validate all resolved parameters - params = validate( - dict( - google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), - timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), - wait=self._get_with_precedence(wait, self.wait, _UNSET), - page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), - extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), - disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), - wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), - wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), - network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), - load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), - selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), - ), - PlaywrightConfig, + params = _validate( + [ + ("google_search", google_search, self.google_search), + ("timeout", timeout, self.timeout), + ("wait", wait, self.wait), + ("page_action", page_action, self.page_action), + ("extra_headers", extra_headers, self.extra_headers), + ("disable_resources", disable_resources, self.disable_resources), + ("wait_selector", wait_selector, self.wait_selector), + ("wait_selector_state", wait_selector_state, self.wait_selector_state), + ("network_idle", network_idle, self.network_idle), + ("load_dom", load_dom, self.load_dom), + ("selector_config", selector_config, self.selector_config), + ], + _UNSET, ) if self._closed: # pragma: no cover diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index ca64942..a2d7c60 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -1,21 +1,69 @@ -from msgspec import Struct, convert, ValidationError -from urllib.parse import urlparse from pathlib import Path +from typing import Annotated +from dataclasses import dataclass +from urllib.parse import urlparse + +from msgspec import Struct, Meta, convert, ValidationError from scrapling.core._types import ( - Optional, Dict, - Callable, List, + Tuple, + Optional, + Callable, SelectorWaitStates, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict +# Custom validators for msgspec +def _validate_file_path(value: str): + """Fast file path validation""" + path = Path(value) + if not path.exists(): + raise ValueError(f"Init script path not found: {value}") + if not path.is_file(): + raise ValueError(f"Init script is not a file: {value}") + if not path.is_absolute(): + raise ValueError(f"Init script is not a absolute path: {value}") + + +def _validate_addon_path(value: str): + """Fast addon path validation""" + path = Path(value) + if not path.exists(): + raise FileNotFoundError(f"Addon path not found: {value}") + if not path.is_dir(): + raise ValueError(f"Addon path must be a directory of the extracted addon: {value}") + + +def _validate_cdp_url(cdp_url: str): + """Fast CDP URL validation""" + try: + # Check the scheme + if not cdp_url.startswith(("ws://", "wss://")): + raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") + + # Validate hostname and port + if not urlparse(cdp_url).netloc: + raise ValueError("Invalid hostname for the CDP URL") + + except AttributeError as e: + raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}") + + except Exception as e: + raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}") + + +# Type aliases for cleaner annotations +PagesCount = Annotated[int, Meta(ge=1, le=50)] +Seconds = Annotated[int, float, Meta(ge=0)] + + class PlaywrightConfig(Struct, kw_only=True, frozen=False): """Configuration struct for validation""" - max_pages: int = 1 + max_pages: PagesCount = 1 cdp_url: Optional[str] = None headless: bool = True google_search: bool = True @@ -23,13 +71,13 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): disable_webgl: bool = False real_chrome: bool = False stealth: bool = False - wait: int | float = 0 + wait: Seconds = 0 page_action: Optional[Callable] = None proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` locale: str = "en-US" extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None - timeout: int | float = 30000 + timeout: Seconds = 30000 init_script: Optional[str] = None disable_resources: bool = False wait_selector: Optional[str] = None @@ -41,52 +89,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): def __post_init__(self): """Custom validation after msgspec validation""" - if self.max_pages < 1 or self.max_pages > 50: - raise ValueError("max_pages must be between 1 and 50") - if self.timeout < 0: - raise ValueError("timeout must be >= 0") if self.page_action and not callable(self.page_action): raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) if self.cdp_url: - self.__validate_cdp(self.cdp_url) + _validate_cdp_url(self.cdp_url) + if not self.cookies: self.cookies = [] if not self.selector_config: self.selector_config = {} if self.init_script is not None: - script_path = Path(self.init_script) - if not script_path.exists(): - raise ValueError("Init script path not found") - elif not script_path.is_file(): - raise ValueError("Init script is not a file") - elif not script_path.is_absolute(): - raise ValueError("Init script is not a absolute path") - - @staticmethod - def __validate_cdp(cdp_url): - try: - # Check the scheme - if not cdp_url.startswith(("ws://", "wss://")): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") - - # Validate hostname and port - if not urlparse(cdp_url).netloc: - raise ValueError("Invalid hostname for the CDP URL") - - except AttributeError as e: - raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}") - - except Exception as e: - raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}") + _validate_file_path(self.init_script) class CamoufoxConfig(Struct, kw_only=True, frozen=False): """Configuration struct for validation""" - max_pages: int = 1 + max_pages: PagesCount = 1 headless: bool = True # noqa: F821 block_images: bool = False disable_resources: bool = False @@ -96,8 +118,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): load_dom: bool = True humanize: bool | float = True solve_cloudflare: bool = False - wait: int | float = 0 - timeout: int | float = 30000 + wait: Seconds = 0 + timeout: Seconds = 30000 init_script: Optional[str] = None page_action: Optional[Callable] = None wait_selector: Optional[str] = None @@ -115,38 +137,23 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): def __post_init__(self): """Custom validation after msgspec validation""" - if self.max_pages < 1 or self.max_pages > 50: - raise ValueError("max_pages must be between 1 and 50") - if self.timeout < 0: - raise ValueError("timeout must be >= 0") if self.page_action and not callable(self.page_action): raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) - if not self.addons: - self.addons = [] - else: + if self.addons and isinstance(self.addons, list): for addon in self.addons: - addon_path = Path(addon) - if not addon_path.exists(): - raise FileNotFoundError(f"Addon's path not found: {addon}") - elif not addon_path.is_dir(): - raise ValueError( - f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}" - ) + _validate_addon_path(addon) + else: + self.addons = [] if self.init_script is not None: - script_path = Path(self.init_script) - if not script_path.exists(): - raise ValueError("Init script path not found") - elif not script_path.is_file(): - raise ValueError("Init script is not a file") - elif not script_path.is_absolute(): - raise ValueError("Init script is not a absolute path") + _validate_file_path(self.init_script) if not self.cookies: self.cookies = [] + # Cloudflare timeout adjustment if self.solve_cloudflare and self.timeout < 60_000: self.timeout = 60_000 if not self.selector_config: @@ -155,10 +162,68 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): self.additional_args = {} -def validate(params, model): - try: - config = convert(params, model) - except ValidationError as e: - raise TypeError(f"Invalid argument type: {e}") +# Code parts to validate `fetch` in the least possible numbers of lines overall +class FetchConfig(Struct, kw_only=True): + """Configuration struct for `fetch` calls validation""" - return config + google_search: bool = True + timeout: Seconds = 30000 + wait: Seconds = 0 + page_action: Optional[Callable] = None + extra_headers: Optional[Dict[str, str]] = None + disable_resources: bool = False + wait_selector: Optional[str] = None + wait_selector_state: SelectorWaitStates = "attached" + network_idle: bool = False + load_dom: bool = True + solve_cloudflare: bool = False + selector_config: Optional[Dict] = {} + + def to_dict(self): + return {f: getattr(self, f) for f in self.__struct_fields__} + + +@dataclass +class _fetch_params: + """A dataclass of all parameters used by `fetch` calls""" + + google_search: bool + timeout: Seconds + wait: Seconds + page_action: Optional[Callable] + extra_headers: Optional[Dict[str, str]] + disable_resources: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + network_idle: bool + load_dom: bool + solve_cloudflare: bool + selector_config: Optional[Dict] + + +def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: + result = {} + overrides = {} + + for arg, request_value, session_value in params: + if request_value is not sentinel: + overrides[arg] = request_value + else: + result[arg] = session_value + + if overrides: + overrides = validate(overrides, FetchConfig).to_dict() + overrides.update(result) + return _fetch_params(**overrides) + + if not result.get("solve_cloudflare"): + result["solve_cloudflare"] = False + + return _fetch_params(**result) + + +def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig: + try: + return convert(params, model) + except ValidationError as e: + raise TypeError(f"Invalid argument type: {e}") from e