From fd9fc83c6cfc3e57a221329380aaa675cccf5b00 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 17 Dec 2025 00:04:24 +0200 Subject: [PATCH 01/11] build: pump version up and deps --- pyproject.toml | 8 ++++---- scrapling/__init__.py | 2 +- scrapling/cli.py | 2 -- setup.cfg | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9e04cc..3084af9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.11" +version = "0.3.12" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "docs/README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -59,14 +59,14 @@ classifiers = [ dependencies = [ "lxml>=6.0.2", "cssselect>=1.3.0", - "orjson>=3.11.4", + "orjson>=3.11.5", "tldextract>=5.3.0", ] [project.optional-dependencies] fetchers = [ "click>=8.3.0", - "curl_cffi>=0.13.0", + "curl_cffi>=0.14.0", "playwright>=1.56.0", "patchright>=1.56.0", "camoufox>=0.4.11", @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.20.0", ] ai = [ - "mcp>=1.23.0", + "mcp>=1.24.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 7cd7812..23f917b 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.11" +__version__ = "0.3.12" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/cli.py b/scrapling/cli.py index 22f2dd1..99a2c63 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,8 +2,6 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable -from curl_cffi.requests import impersonate - from scrapling.core.utils import log from scrapling.engines.toolbelt.custom import Response from scrapling.core.utils._shell import _CookieParser, _ParseHeaders diff --git a/setup.cfg b/setup.cfg index a37bb14..beef70a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.11 +version = 0.3.12 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From ae719a9d543c920cbda56cd66387be079617f92c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 17 Dec 2025 00:15:55 +0200 Subject: [PATCH 02/11] fix(parser): Improve response to json conversion --- scrapling/parser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 95b88ee..f7da3a9 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -939,8 +939,13 @@ class Selector(SelectorsGeneration): # Operations on text functions def json(self) -> Dict: """Return JSON response if the response is jsonable otherwise throws error""" - if self._raw_body and isinstance(self._raw_body, str): - return TextHandler(self._raw_body).json() + if self._raw_body and isinstance(self._raw_body, (str, bytes)): + if isinstance(self._raw_body, str): + return TextHandler(self._raw_body).json() + else: + if TYPE_CHECKING: + assert isinstance(self._raw_body, bytes) + return TextHandler(self._raw_body.decode()).json() elif self.text: return self.text.json() else: From b507e4d4a0c120817a6f5350dd389bbc130c12fc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 17 Dec 2025 01:27:40 +0200 Subject: [PATCH 03/11] refactor(fetchers): rename internal api To make it easier to use to use sessions outside `with` context --- scrapling/engines/_browsers/_base.py | 8 +-- scrapling/engines/_browsers/_camoufox.py | 38 ++++++------ scrapling/engines/_browsers/_controllers.py | 64 +++++++++++---------- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6e5734e..7aa5aeb 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -40,7 +40,7 @@ class SyncSession: self.context: BrowserContext | Any = None self._closed = False - def __create__(self): + def start(self): pass def close(self): # pragma: no cover @@ -59,7 +59,7 @@ class SyncSession: self._closed = True def __enter__(self): - self.__create__() + self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -145,7 +145,7 @@ class AsyncSession: self._closed = False self._lock = Lock() - async def __create__(self): + async def start(self): pass async def close(self): @@ -164,7 +164,7 @@ class AsyncSession: self._closed = True async def __aenter__(self): - await self.__create__() + await self.start() return self async def __aexit__(self, exc_type, exc_val, exc_tb): diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 436a006..e3cbc67 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -102,16 +102,19 @@ class StealthySession(StealthySessionMixin, SyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - def __create__(self): + def start(self): """Create a browser for this instance and context.""" - self.playwright = sync_playwright().start() - self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) + if not self.playwright: + self.playwright = sync_playwright().start() + self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) + else: + raise RuntimeError("Session has been already started") def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -299,18 +302,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - async def __create__(self): + async def start(self): """Create a browser for this instance and context.""" - self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + if not self.playwright: + self.playwright: AsyncPlaywright = await async_playwright().start() + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( + **self.launch_options + ) - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] + else: + raise RuntimeError("Session has been already started") async def _cloudflare_solver(self, page: async_Page): # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed. The async version diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 9ab0e3c..a338c93 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -95,24 +95,27 @@ class DynamicSession(DynamicSessionMixin, SyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - def __create__(self): + def start(self): """Create a browser for this instance and context.""" - sync_context = sync_patchright if self._stealth else sync_playwright + if not self.playwright: + sync_context = sync_patchright if self._stealth else sync_playwright - self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] + self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self._cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( - **self.context_options - ) + if self._cdp_url: # pragma: no cover + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( + **self.context_options + ) + else: + self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) + + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) + + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) else: - self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) - - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) - - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + raise RuntimeError("Session has been already started") def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -227,25 +230,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - async def __create__(self): + async def start(self): """Create a browser for this instance and context.""" - async_context = async_patchright if self._stealth else async_playwright + if not self.playwright: + async_context = async_patchright if self._stealth else async_playwright - self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] + self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self._cdp_url: - browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) - self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) + if self._cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) + else: + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + **self.launch_options + ) + + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) + + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore else: - self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - **self.launch_options - ) - - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) - - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore + raise RuntimeError("Session has been already started") async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. From 39f544eae4404aec3372c2dcfd8f1eaf2eef3c58 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:13:31 +0200 Subject: [PATCH 04/11] feat(dynamicSession): add the option to set timezone of the browser --- scrapling/engines/_browsers/_base.py | 2 ++ scrapling/engines/_browsers/_config_tools.py | 2 ++ scrapling/engines/_browsers/_validators.py | 1 + 3 files changed, 5 insertions(+) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 7aa5aeb..fdb298c 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -281,6 +281,7 @@ class DynamicSessionMixin: self._wait_selector_state = config.wait_selector_state self._extra_flags = config.extra_flags self._selector_config = config.selector_config + self._timezone_id = config.timezone_id self._additional_args = config.additional_args self._page_action = config.page_action self._user_data_dir = config.user_data_dir @@ -304,6 +305,7 @@ class DynamicSessionMixin: self._stealth, self._hide_canvas, self._disable_webgl, + self._timezone_id, tuple(self._extra_flags) if self._extra_flags else tuple(), ) ) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 57e2f17..eaf28d6 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -70,6 +70,7 @@ def _launch_kwargs( stealth, hide_canvas, disable_webgl, + timezone_id, extra_flags: Tuple, ) -> Tuple: """Creates the arguments we will use while launching playwright's browser""" @@ -79,6 +80,7 @@ def _launch_kwargs( launch_kwargs = { "locale": locale, + "timezone_id": timezone_id or None, "headless": headless, "args": base_args, "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index a867a37..f8b9875 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" user_data_dir: str = "" + timezone_id: str = "" extra_flags: Optional[List[str]] = None selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} From 9443ab80ae7df61dfdb41969aa1d998d6d50e1d9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:14:05 +0200 Subject: [PATCH 05/11] test: update tests accordingly --- tests/fetchers/sync/test_dynamic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_dynamic.py index 2b1a54d..e044e33 100644 --- a/tests/fetchers/sync/test_dynamic.py +++ b/tests/fetchers/sync/test_dynamic.py @@ -62,6 +62,7 @@ class TestDynamicFetcher: "real_chrome": True, "wait": 10, "locale": "en-US", + "timezone_id": "America/New_York", "extra_headers": {"ayo": ""}, "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0", "cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}], From ecb97d3fd606318159c8078c4956d1341cbae78e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:16:22 +0200 Subject: [PATCH 06/11] docs: add docs for `timezone_id` argument --- docs/fetching/dynamic.md | 1 + scrapling/engines/_browsers/_controllers.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 066d12c..745f867 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -93,6 +93,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | stealth | Enables stealth mode; you should always check the documentation to see what the stealth mode does currently. | ✔️ | | real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser. | ✔️ | | locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | +| timezone_id | Set the timezone for the browser if wanted. | ✔️ | | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | | extra_flags | A list of additional browser flags to pass to the browser on launch. | ✔️ | diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index a338c93..9e48675 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -77,6 +77,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param timezone_id: Set the timezone for the browser if wanted. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. @@ -212,6 +213,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param timezone_id: Set the timezone for the browser if wanted. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. From 381c4eab7f134aa13a0c8503fb03815b02c0a7fa Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:23:50 +0200 Subject: [PATCH 07/11] test: skip checking camoufox browser version if he cache hits To avoid GitHub repeated rate-limit errors --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 368c9a8..0360904 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,7 +108,11 @@ jobs: - name: Install Camoufox browser run: | echo "Cache hit: ${{ steps.camoufox-cache.outputs.cache-hit }}" - python3 -m camoufox fetch --browserforge + if [ "${{ steps.camoufox-cache.outputs.cache-hit }}" != "true" ]; then + python3 -m camoufox fetch --browserforge + else + echo "Skipping fetch - using cached Camoufox browser" + fi # Cache tox environments - name: Cache tox environments From 1694b44a2b4ad729779b6515452c550f029c31d7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:30:00 +0200 Subject: [PATCH 08/11] test: adjusting deps versions for tox --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 758939f..2a24c42 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,9 @@ envlist = pre-commit,py{310,311,312,313} usedevelop = True changedir = tests deps = - playwright>=1.55.0 - patchright>=1.55.0 - camoufox + playwright>=1.56.0 + patchright>=1.56.0 + camoufox>=0.4.11 -r{toxinidir}/tests/requirements.txt extras = ai,shell commands = From 580bc5390f02c3b3fe3316b735a9ebfd3a3c24f3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:44:05 +0200 Subject: [PATCH 09/11] test: Adjusting test workflow for GitHub --- .github/workflows/tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0360904..f610713 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: - name: Install all browsers dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install playwright>=1.55.0 patchright>=1.55.0 camoufox + python3 -m pip install playwright==1.56.0 patchright==1.56.0 camoufox>=0.4.11 - name: Get Playwright version id: playwright-version @@ -83,7 +83,10 @@ jobs: - name: Install Playwright browsers run: | echo "Cache hit: ${{ steps.playwright-cache.outputs.cache-hit }}" - python3 -m playwright install chromium + if [ "${{ steps.playwright-cache.outputs.cache-hit }}" != "true" ]; then + python3 -m playwright install chromium + else + echo "Skipping install - using cached Playwright browsers" python3 -m playwright install-deps chromium firefox - name: Get Camoufox version From e53f1cb363279fbfdb5ddfdbe88257216e8340c7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:45:21 +0200 Subject: [PATCH 10/11] build: forcing playwright version Until the Python driver for patchright 1.57 releases --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3084af9..0dd663b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,8 +67,8 @@ dependencies = [ fetchers = [ "click>=8.3.0", "curl_cffi>=0.14.0", - "playwright>=1.56.0", - "patchright>=1.56.0", + "playwright==1.56.0", + "patchright==1.56.0", "camoufox>=0.4.11", "geoip2>=5.2.0", "msgspec>=0.20.0", From a7b55d4a746bd75f98ce32e9f7a940b7fa0b22f1 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 18 Dec 2025 01:48:47 +0200 Subject: [PATCH 11/11] test: fix for typo in the workflow --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f610713..ecdfa89 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,6 +87,7 @@ jobs: python3 -m playwright install chromium else echo "Skipping install - using cached Playwright browsers" + fi python3 -m playwright install-deps chromium firefox - name: Get Camoufox version