diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index bf2e5ff..62338f3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,8 +1,10 @@ name: Build and Push Docker Image on: - release: - types: [published] + pull_request: + types: [closed] + branches: + - main workflow_dispatch: inputs: tag: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a67bd50..1a20012 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: args: [-r, -c, .bandit.yml] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.13.0 + rev: v0.13.3 hooks: # Run the linter. - id: ruff diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index bc57dae..3075fdc 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -17,7 +17,7 @@ Now, we will review most of the arguments one by one, using examples. If you wan > Note: The async version of the `fetch` method is the `async_fetch` method, of course. -This fetcher currently provides four main run options, which can be mixed as desired. +This fetcher currently provides four main run options that can be combined as desired. Which are: @@ -62,7 +62,7 @@ DynamicFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222') Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). ## Full list of arguments -Scrapling provides many options with this fetcher and its session classes. To make it as simple as possible, we will list the options here and give examples of using most of them. +Scrapling provides many options with this fetcher and its session classes. To make it as simple as possible, we will list the options here and give examples of how to use most of them. | Argument | Description | Optional | |:-------------------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| @@ -81,13 +81,15 @@ Scrapling provides many options with this fetcher and its session classes. To ma | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | | google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ | -| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | +| proxy | The proxy to be used with requests. It can be a string or a dictionary with only the keys 'server', 'username', and 'password'. | ✔️ | | hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ | | disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ | | stealth | Enables stealth mode; you should always check the documentation to see what the stealth mode does currently. | ✔️ | | real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser. | ✔️ | | locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | +| user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | +| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, and `selector_config`. @@ -123,11 +125,11 @@ page = DynamicFetcher.fetch( ``` ### Browser Automation -This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then returns it for the current fetcher to continue processing. +This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues. -This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for various purposes, not just automation. You can alter the page as you want. +This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for purposes beyond automation. You can alter the page as you want. -In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse. +In the example below, I used the pages' [mouse events](https://playwright.dev/python/docs/api/class-mouse) to scroll the page with the mouse wheel, then move the mouse. ```python from playwright.sync_api import Page @@ -201,9 +203,9 @@ page = DynamicFetcher.fetch( locale='en-US' ) ``` -Hence, the `hide_canvas` argument doesn't disable the canvas but instead hides it by adding random noise to canvas operations, preventing fingerprinting. Also, if you didn't set a user agent (preferred), the fetcher will generate a real User Agent of the same browser and use it. +Hence, the `hide_canvas` argument doesn't disable the canvas; instead, it hides it by adding random noise to canvas operations, preventing fingerprinting. Also, if you didn't set a user agent (preferred), the fetcher will generate a real User Agent of the same browser and use it. -The `google_search` argument is enabled by default, making the request look as if it came from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. +The `google_search` argument is enabled by default, making the request appear to come from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. ### General example ```python @@ -272,14 +274,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: 1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. -2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive. This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) -In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used with the request before this one. +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved flawed, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used in the request before this one. ### Session Benefits diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 998fd27..ddf8410 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -1,6 +1,6 @@ # Introduction -Here, we will discuss the `StealthyFetcher` class. This class is similar to [DynamicFetcher](dynamic.md#introduction) in many ways, such as browser automation and the utilization of [Playwright's API](https://playwright.dev/python/docs/intro). The main difference is that this class provides advanced anti-bot protection bypass capabilities and a custom version of a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), from which most stealth comes. +Here, we will discuss the `StealthyFetcher` class. This class is similar to [DynamicFetcher](dynamic.md#introduction) in many ways, including browser automation and the use of [Playwright's API](https://playwright.dev/python/docs/intro). The main difference is that this class provides advanced anti-bot protection bypass capabilities and a custom version of a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), from which most stealth comes. As with [DynamicFetcher](dynamic.md#introduction), you will need some knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) to automate the page, as we will explain later. @@ -43,14 +43,15 @@ Scrapling provides many options with this fetcher and its session classes. Befor | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | | init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | -| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | +| proxy | The proxy to be used with requests. It can be a string or a dictionary with only the keys 'server', 'username', and 'password'. | ✔️ | +| user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | | additional_args | Additional arguments to be passed to Camoufox as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, and `selector_config`. ## Examples -It's easier to understand with examples, so we will now review most of the arguments individually with examples. +It's easier to understand with examples, so we will now review most of the arguments individually. ### Browser Modes @@ -97,8 +98,11 @@ The `solve_cloudflare` parameter enables automatic detection and solving all typ - Interactive challenges (clicking verification boxes) - Invisible challenges (automatic background verification) +And even solves the custom pages. + **Important notes:** +- Sometimes, with websites that use custom implementations, you will need to use `wait_selector` to make sure Scrapling waits for the real website content to be loaded after solving the captcha. Some websites can be the real definition of an edge case while we are trying to make the solver as generic as possible. - When `solve_cloudflare=True` is enabled, `humanize=True` is automatically activated for more realistic behavior - The timeout should be at least 60 seconds when using the Cloudflare solver for sufficient challenge-solving time - This feature works seamlessly with proxies and other stealth options @@ -124,7 +128,7 @@ page = StealthyFetcher.fetch( ) ``` -The `google_search` argument is enabled by default, making the request look as if it came from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. +The `google_search` argument is enabled by default, making the request appear to come from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. ### Network Control @@ -143,11 +147,11 @@ page = StealthyFetcher.fetch( ``` ### Browser Automation -This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then returns it for the current fetcher to continue processing. +This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues. -This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for various purposes, not just automation. You can alter the page as you want. +This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for purposes beyond automation. You can alter the page as you want. -In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse. +In the example below, I used the pages' [mouse events](https://playwright.dev/python/docs/api/class-mouse) to scroll the page with the mouse wheel, then move the mouse. ```python from playwright.sync_api import Page @@ -205,7 +209,7 @@ page = StealthyFetcher.fetch( addons=['/path/to/addon1', '/path/to/addon2'] ) ``` -The paths here must be paths of extracted addons, which will be installed automatically upon browser launch. +The paths here must point to extracted addons that will be installed automatically upon browser launch. ### Real-world example (Amazon) This is for educational purposes only; this example was generated by AI, which shows how easy it is to work with Scrapling through AI @@ -275,14 +279,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: 1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. -2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive. This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) -In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used with the request before this one. +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved flawed, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used in the request before this one. ### Session Benefits diff --git a/docs/parsing/main_classes.md b/docs/parsing/main_classes.md index ce2c1e8..9d4260d 100644 --- a/docs/parsing/main_classes.md +++ b/docs/parsing/main_classes.md @@ -149,11 +149,6 @@ Get the HTML content of the element >>> article.html_content '

Product 1

\n

This is product 1

\n $10.99\n \n
' ``` -It's the same if you used the `.body` property -```python ->>> article.body -'

Product 1

\n

This is product 1

\n $10.99\n \n
' -``` Get the prettified version of the element's HTML content ```python >>> print(article.prettify()) @@ -163,6 +158,11 @@ Get the prettified version of the element's HTML content ``` +Use `.body` property to get the raw content of page +```python +>>> page.body +'\n \n Some page\n \n \n
\n
\n

Product 1

\n

This is product 1

\n $10.99\n \n
\n\n
\n

Product 2

\n

This is product 2

\n $20.99\n \n
\n\n
\n

Product 3

\n

This is product 3

\n $15.99\n \n
\n
\n\n \n \n' +``` To get all the ancestors in the DOM tree of this element ```python >>> article.path diff --git a/pyproject.toml b/pyproject.toml index 8b2a555..252f05d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.6" +version = "0.3.7" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.19.0", ] ai = [ - "mcp>=1.15.0", + "mcp>=1.16.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index d30a8db..cbc3a2c 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.6" +__version__ = "0.3.7" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 59422f2..51016d4 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -12,9 +12,11 @@ from typing import ( Generator, Iterable, List, + Set, Literal, Optional, Pattern, + Sequence, Tuple, TypeVar, Union, @@ -22,6 +24,7 @@ from typing import ( Mapping, Awaitable, Protocol, + Coroutine, SupportsIndex, ) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 283f183..d497e57 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -20,6 +20,7 @@ from scrapling.core._types import ( Mapping, Dict, List, + Any, SelectorWaitStates, Generator, ) @@ -171,7 +172,7 @@ class ScraplingMCPServer: :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ async with FetcherSession() as session: - tasks = [ + tasks: List[Any] = [ session.get( url, auth=auth, diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index e9ece7d..3675372 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -5,6 +5,7 @@ from re import compile as re_compile, UNICODE, IGNORECASE from orjson import dumps, loads from scrapling.core._types import ( + Any, cast, Dict, List, @@ -14,7 +15,6 @@ from scrapling.core._types import ( Literal, Pattern, Iterable, - Optional, Generator, SupportsIndex, ) @@ -33,23 +33,20 @@ class TextHandler(str): def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": # pragma: no cover lst = super().__getitem__(key) - return cast(_TextHandlerType, TextHandler(lst)) + return TextHandler(lst) - def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": # pragma: no cover - return TextHandlers( - cast( - List[_TextHandlerType], - [TextHandler(s) for s in super().split(sep, maxsplit)], - ) - ) + def split( + self, sep: str | None = None, maxsplit: SupportsIndex = -1 + ) -> Union[List, "TextHandlers"]: # pragma: no cover + return TextHandlers([TextHandler(s) for s in super().split(sep, maxsplit)]) - def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover + def strip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().strip(chars)) - def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover + def lstrip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().lstrip(chars)) - def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover + def rstrip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().rstrip(chars)) def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover @@ -64,7 +61,7 @@ class TextHandler(str): def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().expandtabs(tabsize)) - def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover + def format(self, *args: object, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format(*args, **kwargs)) def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover @@ -131,10 +128,11 @@ class TextHandler(str): def re( self, regex: str | Pattern, - check_match: Literal[True], replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, + *, + check_match: Literal[True], ) -> bool: ... @overload @@ -179,19 +177,14 @@ class TextHandler(str): results = flatten(results) if not replace_entities: - return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results])) + return TextHandlers([TextHandler(string) for string in results]) - return TextHandlers( - cast( - List[_TextHandlerType], - [TextHandler(_replace_entities(s)) for s in results], - ) - ) + return TextHandlers([TextHandler(_replace_entities(s)) for s in results]) def re_first( self, regex: str | Pattern, - default=None, + default: Any = None, replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, @@ -232,8 +225,8 @@ class TextHandlers(List[TextHandler]): def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]: lst = super().__getitem__(pos) if isinstance(pos, slice): - return TextHandlers(cast(List[_TextHandlerType], lst)) - return cast(_TextHandlerType, TextHandler(lst)) + return TextHandlers(cast(List[TextHandler], lst)) + return TextHandler(cast(TextHandler, lst)) def re( self, @@ -256,7 +249,7 @@ class TextHandlers(List[TextHandler]): def re_first( self, regex: str | Pattern, - default=None, + default: Any = None, replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, @@ -309,9 +302,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): ) # Fastest read-only mapping type - self._data = MappingProxyType(mapping) + self._data: Mapping[str, Any] = MappingProxyType(mapping) - def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]: + def get(self, key: str, default: Any = None) -> _TextHandlerType: """Acts like the standard dictionary `.get()` method""" return self._data.get(key, default) diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py index 4087020..3a96bda 100644 --- a/scrapling/core/mixins.py +++ b/scrapling/core/mixins.py @@ -1,3 +1,9 @@ +from scrapling.core._types import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapling.parser import Selector + + class SelectorsGeneration: """ Functions for generating selectors @@ -5,7 +11,7 @@ class SelectorsGeneration: Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591 """ - def __general_selection(self, selection: str = "css", full_path: bool = False) -> str: + def _general_selection(self: "Selector", selection: str = "css", full_path: bool = False) -> str: # type: ignore[name-defined] """Generate a selector for the current element. :return: A string of the generated selector. """ @@ -47,29 +53,29 @@ class SelectorsGeneration: return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) @property - def generate_css_selector(self) -> str: + def generate_css_selector(self: "Selector") -> str: # type: ignore[name-defined] """Generate a CSS selector for the current element :return: A string of the generated selector. """ - return self.__general_selection() + return self._general_selection() @property - def generate_full_css_selector(self) -> str: + def generate_full_css_selector(self: "Selector") -> str: # type: ignore[name-defined] """Generate a complete CSS selector for the current element :return: A string of the generated selector. """ - return self.__general_selection(full_path=True) + return self._general_selection(full_path=True) @property - def generate_xpath_selector(self) -> str: + def generate_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined] """Generate an XPath selector for the current element :return: A string of the generated selector. """ - return self.__general_selection("xpath") + return self._general_selection("xpath") @property - def generate_full_xpath_selector(self) -> str: + def generate_full_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined] """Generate a complete XPath selector for the current element :return: A string of the generated selector. """ - return self.__general_selection("xpath", full_path=True) + return self._general_selection("xpath", full_path=True) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 5ef4458..0799496 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -31,6 +31,7 @@ from scrapling.core._types import ( Optional, Dict, Any, + cast, extraction_types, Generator, ) @@ -540,15 +541,15 @@ class Convertor: raise ValueError(f"Unknown extraction type: {extraction_type}") else: if main_content_only: - page = page.css_first("body") or page + page = cast(Selector, page.css_first("body")) or page - pages = [page] if not css_selector else page.css(css_selector) + pages = [page] if not css_selector else cast(Selectors, page.css(css_selector)) for page in pages: match extraction_type: case "markdown": yield cls._convert_to_markdown(page.html_content) case "html": - yield page.body + yield page.html_content case "text": txt_content = page.get_all_text(strip=True) for s in ( diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index e832cf0..50258c5 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -56,13 +56,13 @@ class StorageSystemMixin(ABC): # pragma: no cover @lru_cache(128, typed=True) def _get_hash(identifier: str) -> str: """If you want to hash identifier in your storage system, use this safer""" - identifier = identifier.lower().strip() - if isinstance(identifier, str): + _identifier = identifier.lower().strip() + if isinstance(_identifier, str): # Hash functions have to take bytes - identifier = identifier.encode("utf-8") + _identifier = _identifier.encode("utf-8") - hash_value = sha256(identifier).hexdigest() - return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance + hash_value = sha256(_identifier).hexdigest() + return f"{hash_value}_{len(_identifier)}" # Length to reduce collision chance @lru_cache(1, typed=True) diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index d98092e..bb6d405 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -10,24 +10,23 @@ So you don't have to learn a new selectors/api method like what bs4 done with so from functools import lru_cache -from cssselect.xpath import ExpressionError -from cssselect.xpath import XPathExpr as OriginalXPathExpr from cssselect import HTMLTranslator as OriginalHTMLTranslator +from cssselect.xpath import ExpressionError, XPathExpr as OriginalXPathExpr from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement -from scrapling.core._types import Any, Optional, Protocol, Self +from scrapling.core._types import Any, Protocol, Self class XPathExpr(OriginalXPathExpr): textnode: bool = False - attribute: Optional[str] = None + attribute: str | None = None @classmethod def from_xpath( cls, xpath: OriginalXPathExpr, textnode: bool = False, - attribute: Optional[str] = None, + attribute: str | None = None, ) -> Self: x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) x.textnode = textnode @@ -71,10 +70,10 @@ class XPathExpr(OriginalXPathExpr): # e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator class TranslatorProtocol(Protocol): - def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pragma: no cover + def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pyright: ignore # pragma: no cover pass - def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pragma: no cover + def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pyright: ignore # pragma: no cover pass @@ -121,9 +120,15 @@ class TranslatorMixin: class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator): - @lru_cache(maxsize=256) def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: return super().css_to_xpath(css, prefix) translator = HTMLTranslator() +# Using a function instead of the translator directly to avoid Pyright override error + + +@lru_cache(maxsize=256) +def css_to_xpath(query: str) -> str: + """Return translated XPath version of a given CSS query""" + return translator.css_to_xpath(query) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index d1a872d..9a6dfae 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -7,14 +7,12 @@ from playwright.async_api import ( BrowserContext as AsyncBrowserContext, Playwright as AsyncPlaywright, ) -from camoufox.utils import ( - launch_options as generate_launch_options, - installed_verstr as camoufox_version, -) +from camoufox.pkgman import installed_verstr as camoufox_version +from camoufox.utils import launch_options as generate_launch_options from ._page import PageInfo, PagePool from scrapling.parser import Selector -from scrapling.core._types import Dict, Optional +from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING from scrapling.engines.toolbelt.fingerprints import get_os_name from ._validators import validate, PlaywrightConfig, CamoufoxConfig from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs @@ -41,6 +39,7 @@ class SyncSession: """Get a new page to use""" # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc. + assert self.context is not None, "Browser context not initialized" page = self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) @@ -65,11 +64,14 @@ class SyncSession: } -class AsyncSession(SyncSession): +class AsyncSession: def __init__(self, max_pages: int = 1): - super().__init__(max_pages) + self.max_pages = max_pages + self.page_pool = PagePool(max_pages) + self._max_wait_for_page = 60 self.playwright: Optional[AsyncPlaywright] = None self.context: Optional[AsyncBrowserContext] = None + self._closed = False self._lock = Lock() async def _get_page( @@ -79,6 +81,9 @@ class AsyncSession(SyncSession): disable_resources: bool, ) -> PageInfo: # pragma: no cover """Get a new page to use""" + if TYPE_CHECKING: + assert self.context is not None, "Browser context not initialized" + async with self._lock: # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: @@ -92,6 +97,7 @@ class AsyncSession(SyncSession): f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" ) + assert self.context is not None, "Browser context not initialized" page = await self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) @@ -107,6 +113,14 @@ class AsyncSession(SyncSession): return self.page_pool.add_page(page) + def get_pool_stats(self) -> Dict[str, int]: + """Get statistics about the current page pool""" + return { + "total_pages": self.page_pool.pages_count, + "busy_pages": self.page_pool.busy_count, + "max_pages": self.max_pages, + } + class DynamicSessionMixin: def __validate__(self, **params): @@ -134,11 +148,16 @@ class DynamicSessionMixin: self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state self.selector_config = config.selector_config + self.additional_args = config.additional_args self.page_action = config.page_action - self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): + if TYPE_CHECKING: + assert isinstance(self.proxy, tuple) + if not self.cdp_url: # `launch_options` is used with persistent context self.launch_options = dict( @@ -156,6 +175,8 @@ class DynamicSessionMixin: ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + self.launch_options["user_data_dir"] = self.user_data_dir + self.launch_options.update(cast(Dict, self.additional_args)) self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled @@ -171,11 +192,12 @@ class DynamicSessionMixin: ) self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + self.context_options.update(cast(Dict, self.additional_args)) class StealthySessionMixin: def __validate__(self, **params): - config = validate(params, model=CamoufoxConfig) + config: CamoufoxConfig = validate(params, model=CamoufoxConfig) self.max_pages = config.max_pages self.headless = config.headless @@ -204,15 +226,16 @@ class StealthySessionMixin: self.selector_config = config.selector_config self.additional_args = config.additional_args self.page_action = config.page_action - self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): """Initiate browser options.""" - self.launch_options = generate_launch_options( + self.launch_options: Dict[str, Any] = generate_launch_options( **{ "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy else self.proxy, + "proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy, "addons": self.addons, "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], "headless": self.headless, @@ -222,7 +245,7 @@ class StealthySessionMixin: "block_webrtc": self.block_webrtc, "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. "os": None if self.os_randomize else get_os_name(), - "user_data_dir": "", + "user_data_dir": self.user_data_dir, "ff_version": __ff_version_str__, "firefox_user_prefs": { # This is what enabling `enable_cache` does internally, so we do it from here instead @@ -232,7 +255,7 @@ class StealthySessionMixin: "browser.cache.disk_cache_ssl": True, "browser.cache.disk.smart_size.enabled": True, }, - **self.additional_args, + **cast(Dict, self.additional_args), } ) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 519ae83..94e8d37 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -1,3 +1,4 @@ +from random import randint from re import compile as re_compile from playwright.sync_api import ( @@ -20,10 +21,12 @@ from ._validators import validate_fetch as _validate from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( + Any, Dict, List, Optional, Callable, + TYPE_CHECKING, SelectorWaitStates, ) from scrapling.engines.toolbelt.convertor import ( @@ -33,7 +36,7 @@ from scrapling.engines.toolbelt.convertor import ( from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -_UNSET = object() +_UNSET: Any = object() class StealthySession(StealthySessionMixin, SyncSession): @@ -101,6 +104,7 @@ class StealthySession(StealthySessionMixin, SyncSession): os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, + user_data_dir: str = "", selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -133,6 +137,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -156,6 +161,7 @@ class StealthySession(StealthySessionMixin, SyncSession): block_images=block_images, block_webrtc=block_webrtc, os_randomize=os_randomize, + user_data_dir=user_data_dir, wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, @@ -170,9 +176,7 @@ class StealthySession(StealthySessionMixin, SyncSession): def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() - self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover - **self.launch_options - ) + self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -203,9 +207,9 @@ class StealthySession(StealthySessionMixin, SyncSession): self._closed = True @staticmethod - def _get_page_content(page: Page) -> str | None: + def _get_page_content(page: Page) -> str: """ - A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. :return: """ @@ -215,6 +219,7 @@ class StealthySession(StealthySessionMixin, SyncSession): except PlaywrightError: page.wait_for_timeout(1000) continue + return "" # pyright: ignore def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -222,6 +227,10 @@ class StealthySession(StealthySessionMixin, SyncSession): :param page: The targeted page :return: """ + try: + page.wait_for_load_state("networkidle", timeout=5000) + except PlaywrightError: + pass challenge_type = self._detect_cloudflare(self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") @@ -244,26 +253,35 @@ class StealthySession(StealthySessionMixin, SyncSession): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared page.wait_for_timeout(500) + outer_box = {} iframe = page.frame(url=__CF_PATTERN__) - if iframe is None: - log.error("Didn't find Cloudflare iframe!") - return + if iframe is not None: + iframe.wait_for_load_state(state="domcontentloaded") + iframe.wait_for_load_state("networkidle") - if challenge_type != "embedded": - while not iframe.frame_element().is_visible(): - # Double-checking that the iframe is loaded - page.wait_for_timeout(500) + if challenge_type != "embedded": + while not iframe.frame_element().is_visible(): + # Double-checking that the iframe is loaded + page.wait_for_timeout(500) + outer_box: Any = iframe.frame_element().bounding_box() + + if not iframe or not outer_box: + outer_box: Any = page.locator(box_selector).last.bounding_box() - iframe.wait_for_load_state(state="domcontentloaded") - iframe.wait_for_load_state("networkidle") # Calculate the Captcha coordinates for any viewport - outer_box = page.locator(box_selector).last.bounding_box() - captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 + captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) # Move the mouse to the center of the window, then press and hold the left mouse button page.mouse.click(captcha_x, captcha_y, delay=60, button="left") + page.wait_for_load_state("networkidle") + if iframe is not None: + # Wait for the frame to be removed from the page + while iframe in page.frames: + page.wait_for_timeout(100) if challenge_type != "embedded": + page.locator(box_selector).last.wait_for(state="detached") page.locator(".zone-name-title").wait_for(state="hidden") + page.wait_for_load_state(state="load") page.wait_for_load_state(state="domcontentloaded") log.info("Cloudflare captcha is solved") @@ -335,6 +353,7 @@ class StealthySession(StealthySessionMixin, SyncSession): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame ): final_response = finished_response @@ -387,7 +406,7 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources page_info.page.close() self.page_pool.pages.remove(page_info) @@ -427,6 +446,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, + user_data_dir: str = "", selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -460,6 +480,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -485,6 +506,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, + user_data_dir=user_data_dir, additional_args=additional_args, selector_config=selector_config, solve_cloudflare=solve_cloudflare, @@ -504,7 +526,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await self.context.add_init_script(path=self.init_script) if self.cookies: - await self.context.add_cookies(self.cookies) + await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType] async def __aenter__(self): await self.__create__() @@ -520,18 +542,18 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if self.context: await self.context.close() - self.context = None + self.context = None # pyright: ignore if self.playwright: await self.playwright.stop() - self.playwright = None + self.playwright = None # pyright: ignore self._closed = True @staticmethod - async def _get_page_content(page: async_Page) -> str | None: + async def _get_page_content(page: async_Page) -> str: """ - A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. :return: """ @@ -541,6 +563,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): except PlaywrightError: await page.wait_for_timeout(1000) continue + return "" # pyright: ignore async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version @@ -548,6 +571,10 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param page: The async targeted page :return: """ + try: + await page.wait_for_load_state("networkidle", timeout=5000) + except PlaywrightError: + pass challenge_type = self._detect_cloudflare(await self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") @@ -570,26 +597,35 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared await page.wait_for_timeout(500) + outer_box = {} iframe = page.frame(url=__CF_PATTERN__) - if iframe is None: - log.error("Didn't find Cloudflare iframe!") - return + if iframe is not None: + await iframe.wait_for_load_state(state="domcontentloaded") + await iframe.wait_for_load_state("networkidle") - if challenge_type != "embedded": - while not await (await iframe.frame_element()).is_visible(): - # Double-checking that the iframe is loaded - await page.wait_for_timeout(500) + if challenge_type != "embedded": + while not await (await iframe.frame_element()).is_visible(): + # Double-checking that the iframe is loaded + await page.wait_for_timeout(500) + outer_box: Any = await (await iframe.frame_element()).bounding_box() + + if not iframe or not outer_box: + outer_box: Any = await page.locator(box_selector).last.bounding_box() - await iframe.wait_for_load_state(state="domcontentloaded") - await iframe.wait_for_load_state("networkidle") # Calculate the Captcha coordinates for any viewport - outer_box = await page.locator(box_selector).last.bounding_box() - captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 + captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) # Move the mouse to the center of the window, then press and hold the left mouse button await page.mouse.click(captcha_x, captcha_y, delay=60, button="left") + await page.wait_for_load_state("networkidle") + if iframe is not None: + # Wait for the frame to be removed from the page + while iframe in page.frames: + await page.wait_for_timeout(100) if challenge_type != "embedded": + await page.locator(box_selector).wait_for(state="detached") await page.locator(".zone-name-title").wait_for(state="hidden") + await page.wait_for_load_state(state="load") await page.wait_for_load_state(state="domcontentloaded") log.info("Cloudflare captcha is solved") @@ -661,12 +697,17 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame ): final_response = finished_response page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) + if TYPE_CHECKING: + if not isinstance(page_info.page, async_Page): + raise TypeError + try: # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) @@ -715,7 +756,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources await page_info.page.close() self.page_pool.pages.remove(page_info) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 4b405e6..322d7ad 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -62,7 +62,7 @@ def _set_flags(hide_canvas, disable_webgl): # pragma: no cover @lru_cache(2, typed=True) def _launch_kwargs( headless, - proxy, + proxy: Tuple, locale, extra_headers, useragent, diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index b895f9a..ca8b45e 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -10,6 +10,7 @@ from playwright.async_api import ( BrowserContext as AsyncBrowserContext, Playwright as AsyncPlaywright, Locator as AsyncLocator, + Page as async_Page, ) from patchright.sync_api import sync_playwright as sync_patchright from patchright.async_api import async_playwright as async_patchright @@ -18,10 +19,12 @@ from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin from ._validators import validate_fetch as _validate from scrapling.core._types import ( + Any, Dict, List, Optional, Callable, + TYPE_CHECKING, SelectorWaitStates, ) from scrapling.engines.toolbelt.convertor import ( @@ -30,7 +33,7 @@ from scrapling.engines.toolbelt.convertor import ( ) from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -_UNSET = object() +_UNSET: Any = object() class DynamicSession(DynamicSessionMixin, SyncSession): @@ -94,7 +97,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + user_data_dir: str = "", selector_config: Optional[Dict] = None, + additional_args: Optional[Dict] = None, ): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. @@ -121,7 +126,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__( wait=wait, @@ -140,11 +147,13 @@ class DynamicSession(DynamicSessionMixin, SyncSession): hide_canvas=hide_canvas, init_script=init_script, network_idle=network_idle, + user_data_dir=user_data_dir, google_search=google_search, extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, selector_config=selector_config, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) @@ -154,14 +163,14 @@ class DynamicSession(DynamicSessionMixin, SyncSession): """Create a browser for this instance and context.""" sync_context = sync_patchright if self.stealth else sync_playwright - self.playwright: Playwright = sync_context().start() + self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] if self.cdp_url: # pragma: no cover self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( **self.context_options ) else: - self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options) + self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -187,7 +196,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): if self.playwright: self.playwright.stop() - self.playwright = None + self.playwright = None # pyright: ignore self._closed = True @@ -254,6 +263,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame ): final_response = finished_response @@ -299,7 +309,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources page_info.page.close() self.page_pool.pages.remove(page_info) @@ -337,7 +347,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + user_data_dir: str = "", selector_config: Optional[Dict] = None, + additional_args: Optional[Dict] = None, ): """A Browser session manager with page pooling @@ -365,7 +377,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__( @@ -385,11 +399,13 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): hide_canvas=hide_canvas, init_script=init_script, network_idle=network_idle, + user_data_dir=user_data_dir, google_search=google_search, extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, selector_config=selector_config, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) @@ -399,21 +415,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """Create a browser for this instance and context.""" async_context = async_patchright if self.stealth else async_playwright - self.playwright: AsyncPlaywright = await async_context().start() + self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] if self.cdp_url: browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options + **self.launch_options ) if self.init_script: # pragma: no cover await self.context.add_init_script(path=self.init_script) if self.cookies: - await self.context.add_cookies(self.cookies) + await self.context.add_cookies(self.cookies) # pyright: ignore async def __aenter__(self): await self.__create__() @@ -429,11 +445,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): if self.context: await self.context.close() - self.context = None + self.context = None # pyright: ignore if self.playwright: await self.playwright.stop() - self.playwright = None + self.playwright = None # pyright: ignore self._closed = True @@ -500,12 +516,17 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame ): final_response = finished_response page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) + if TYPE_CHECKING: + if not isinstance(page_info.page, async_Page): + raise TypeError + try: # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) @@ -545,7 +566,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources await page_info.page.close() self.page_pool.pages.remove(page_info) return response diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index a2d7c60..0d7d236 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -11,7 +11,9 @@ from scrapling.core._types import ( Tuple, Optional, Callable, + Iterable, SelectorWaitStates, + overload, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict @@ -73,7 +75,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): stealth: bool = False wait: Seconds = 0 page_action: Optional[Callable] = None - proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` + proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` locale: str = "en-US" extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None @@ -81,11 +83,13 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): init_script: Optional[str] = None disable_resources: bool = False wait_selector: Optional[str] = None - cookies: Optional[List[Dict]] = None + cookies: Optional[Iterable[Dict]] = None network_idle: bool = False load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" - selector_config: Optional[Dict] = None + user_data_dir: str = "" + selector_config: Optional[Dict] = {} + additional_args: Optional[Dict] = {} def __post_init__(self): """Custom validation after msgspec validation""" @@ -100,6 +104,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): self.cookies = [] if not self.selector_config: self.selector_config = {} + if not self.additional_args: + self.additional_args = {} if self.init_script is not None: _validate_file_path(self.init_script) @@ -125,15 +131,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): wait_selector: Optional[str] = None addons: Optional[List[str]] = None wait_selector_state: SelectorWaitStates = "attached" - cookies: Optional[List[Dict]] = None + cookies: Optional[Iterable[Dict]] = None google_search: bool = True extra_headers: Optional[Dict[str, str]] = None - proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` + proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` os_randomize: bool = False disable_ads: bool = False geoip: bool = False - selector_config: Optional[Dict] = None - additional_args: Optional[Dict] = None + user_data_dir: str = "" + selector_config: Optional[Dict] = {} + additional_args: Optional[Dict] = {} def __post_init__(self): """Custom validation after msgspec validation""" @@ -177,7 +184,7 @@ class FetchConfig(Struct, kw_only=True): network_idle: bool = False load_dom: bool = True solve_cloudflare: bool = False - selector_config: Optional[Dict] = {} + selector_config: Dict = {} def to_dict(self): return {f: getattr(self, f) for f in self.__struct_fields__} @@ -198,7 +205,7 @@ class _fetch_params: network_idle: bool load_dom: bool solve_cloudflare: bool - selector_config: Optional[Dict] + selector_config: Dict def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: @@ -222,7 +229,21 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: return _fetch_params(**result) -def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig: +@overload +def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... + + +@overload +def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ... + + +@overload +def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ... + + +def validate( + params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig] +) -> PlaywrightConfig | CamoufoxConfig | FetchConfig: try: return convert(params, model) except ValidationError as e: diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 3c8bef9..dd54168 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,9 +1,9 @@ +from abc import ABC from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep from curl_cffi.curl import CurlError from curl_cffi import CurlHttpVersion -from curl_cffi.requests.impersonate import DEFAULT_CHROME from curl_cffi.requests import ( ProxySpec, CookieTypes, @@ -22,7 +22,6 @@ from scrapling.core._types import ( Awaitable, List, Any, - cast, ) from .toolbelt.custom import Response @@ -30,11 +29,937 @@ from .toolbelt.convertor import ResponseFactory from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ _UNSET: Any = object() +_NO_SESSION: Any = object() + + +class _ConfigurationLogic(ABC): + # Core Logic Handler (Internal Engine) + def __init__( + self, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + self._default_impersonate = impersonate + self._stealth = stealthy_headers + self._default_proxies = proxies or {} + self._default_proxy = proxy or None + self._default_proxy_auth = proxy_auth or None + self._default_timeout = timeout + self._default_headers = headers or {} + self._default_retries = retries + self._default_retry_delay = retry_delay + self._default_follow_redirects = follow_redirects + self._default_max_redirects = max_redirects + self._default_verify = verify + self._default_cert = cert + self._default_http3 = http3 + self.selector_config = selector_config or {} + + @staticmethod + def _get_with_precedence(request_val: Any, default_val: Any) -> Any: + """Get value with request-level priority over session-level""" + return request_val if request_val is not _UNSET else default_val + + def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: + """Merge request-specific arguments with default session arguments.""" + url = method_kwargs.pop("url") + impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) + http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3) + final_args = { + "url": url, + # Curl automatically generates the suitable browser headers when you use `impersonate` + "headers": self._headers_job( + url, + self._get_with_precedence(method_kwargs.pop("headers"), self._default_headers), + self._get_with_precedence(method_kwargs.pop("stealth"), self._stealth), + bool(impersonate), + ), + "proxies": self._get_with_precedence(method_kwargs.pop("proxies"), self._default_proxies), + "proxy": self._get_with_precedence(method_kwargs.pop("proxy"), self._default_proxy), + "proxy_auth": self._get_with_precedence(method_kwargs.pop("proxy_auth"), self._default_proxy_auth), + "timeout": self._get_with_precedence(method_kwargs.pop("timeout"), self._default_timeout), + "allow_redirects": self._get_with_precedence( + method_kwargs.pop("follow_redirects"), self._default_follow_redirects + ), + "max_redirects": self._get_with_precedence(method_kwargs.pop("max_redirects"), self._default_max_redirects), + "verify": self._get_with_precedence(method_kwargs.pop("verify"), self._default_verify), + "cert": self._get_with_precedence(method_kwargs.pop("cert"), self._default_cert), + "impersonate": impersonate, + **{ + k: v + for k, v in method_kwargs.items() + if v + not in ( + _UNSET, + None, + ) + }, # Add any remaining parameters (after all known ones are popped) + } + if http3_enabled: # pragma: no cover + final_args["http_version"] = CurlHttpVersion.V3ONLY + if impersonate: + log.warning( + "The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors." + ) + + return final_args + + def _headers_job(self, url, headers: Dict, stealth: bool, impersonate_enabled: bool) -> Dict: + """ + 1. Adds a useragent to the headers if it doesn't have one + 2. Generates real headers and append them to current headers + 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain. + """ + # Merge session headers with request headers, request takes precedence (if it was set) + final_headers = {**self._default_headers, **(headers if headers and headers is not _UNSET else {})} + headers_keys = {k.lower() for k in final_headers} + if stealth: + if "referer" not in headers_keys: + final_headers["referer"] = generate_convincing_referer(url) + + if not impersonate_enabled: # Curl will generate the suitable headers + extra_headers = generate_headers(browser_mode=False) + final_headers.update( + {k: v for k, v in extra_headers.items() if k.lower() not in headers_keys} + ) # Don't overwrite user-supplied headers + + elif "user-agent" not in headers_keys and not impersonate_enabled: + final_headers["User-Agent"] = __default_useragent__ + log.debug(f"Can't find useragent in headers so '{final_headers['User-Agent']}' was used.") + + return final_headers + + +class _SyncSessionLogic(_ConfigurationLogic): + def __init__( + self, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + super().__init__( + impersonate, + http3, + stealthy_headers, + proxies, + proxy, + proxy_auth, + timeout, + headers, + retries, + retry_delay, + follow_redirects, + max_redirects, + verify, + cert, + selector_config, + ) + self._curl_session: Optional[CurlSession] = None + + def __enter__(self): + """Creates and returns a new synchronous Fetcher Session""" + if self._curl_session: + raise RuntimeError("This FetcherSession instance already has an active synchronous session.") + + self._curl_session = CurlSession() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Closes the active synchronous session managed by this instance, if any.""" + # For type checking (not accessed error) + _ = ( + exc_type, + exc_val, + exc_tb, + ) + if self._curl_session: + self._curl_session.close() + self._curl_session = None + + def __make_request( + self, + method: SUPPORTED_HTTP_METHODS, + stealth: Optional[bool] = None, + **kwargs, + ) -> Response: + """ + Perform an HTTP request using the configured session. + """ + stealth = self._stealth if stealth is None else stealth + + selector_config = kwargs.pop("selector_config", {}) or self.selector_config + max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) + retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + request_args = self._merge_request_args(stealth=stealth, **kwargs) + + session = self._curl_session + one_off_request = False + if session is _NO_SESSION and self.__enter__ is None: + # For usage inside FetcherClient + # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. + session = CurlSession() + one_off_request = True + + if session: + for attempt in range(max_retries): + try: + response = session.request(method, **request_args) + result = ResponseFactory.from_http_request(response, selector_config) + return result + except CurlError as e: # pragma: no cover + if attempt < max_retries - 1: + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") + time_sleep(retry_delay) + else: + log.error(f"Failed after {max_retries} attempts: {e}") + raise # Raise the exception if all retries fail + finally: + if session and one_off_request: + session.close() + + raise RuntimeError("No active session available.") # pragma: no cover + + def get( + self, + url: str, + params: Optional[Dict | List | Tuple] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Response: + """ + Perform a GET request. + + :param url: Target URL for the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("GET", stealth=stealthy_headers, **method_args) + + def post( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Response: + """ + Perform a POST request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("POST", stealth=stealthy_headers, **method_args) + + def put( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Response: + """ + Perform a PUT request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("PUT", stealth=stealthy_headers, **method_args) + + def delete( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Response: + """ + Perform a DELETE request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) + + +class _ASyncSessionLogic(_ConfigurationLogic): + def __init__( + self, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + super().__init__( + impersonate, + http3, + stealthy_headers, + proxies, + proxy, + proxy_auth, + timeout, + headers, + retries, + retry_delay, + follow_redirects, + max_redirects, + verify, + cert, + selector_config, + ) + self._async_curl_session: Optional[AsyncCurlSession] = None + + async def __aenter__(self): + """Creates and returns a new asynchronous Session.""" + if self._async_curl_session: + raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") + + self._async_curl_session = AsyncCurlSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Closes the active asynchronous session managed by this instance, if any.""" + # For type checking (not accessed error) + _ = ( + exc_type, + exc_val, + exc_tb, + ) + if self._async_curl_session: + await self._async_curl_session.close() + self._async_curl_session = None + + async def __make_request( + self, + method: SUPPORTED_HTTP_METHODS, + stealth: Optional[bool] = None, + **kwargs, + ) -> Response: + """ + Perform an HTTP request using the configured session. + """ + stealth = self._stealth if stealth is None else stealth + + selector_config = kwargs.pop("selector_config", {}) or self.selector_config + max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) + retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + request_args = self._merge_request_args(stealth=stealth, **kwargs) + + session = self._async_curl_session + one_off_request = False + if session is _NO_SESSION and self.__aenter__ is None: + # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons + # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. + # 2. `curl_cffi` doesn't support making async requests without sessions + # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi. + session = AsyncCurlSession() + one_off_request = True + + if session: + for attempt in range(max_retries): + try: + response = await session.request(method, **request_args) + result = ResponseFactory.from_http_request(response, selector_config) + return result + except CurlError as e: # pragma: no cover + if attempt < max_retries - 1: + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") + await asyncio_sleep(retry_delay) + else: + log.error(f"Failed after {max_retries} attempts: {e}") + raise # Raise the exception if all retries fail + finally: + if session and one_off_request: + await session.close() + + raise RuntimeError("No active session available.") # pragma: no cover + + def get( + self, + url: str, + params: Optional[Dict | List | Tuple] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Awaitable[Response]: + """ + Perform a GET request. + + :param url: Target URL for the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("GET", stealth=stealthy_headers, **method_args) + + def post( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Awaitable[Response]: + """ + Perform a POST request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("POST", stealth=stealthy_headers, **method_args) + + def put( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Awaitable[Response]: + """ + Perform a PUT request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("PUT", stealth=stealthy_headers, **method_args) + + def delete( + self, + url: str, + data: Optional[Dict | str] = None, + json: Optional[Dict | List] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, + params: Optional[Dict | List | Tuple] = None, + cookies: Optional[CookieTypes] = None, + timeout: Optional[int | float] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = _UNSET, + cert: Optional[str | Tuple[str, str]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, + **kwargs, + ) -> Awaitable[Response]: + """ + Perform a DELETE request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) class FetcherSession: """ - A context manager that provides configured Fetcher sessions. + A factory context manager that provides configured Fetcher sessions. When this manager is used in a 'with' or 'async with' block, it yields a new session configured with the manager's defaults. @@ -45,7 +970,7 @@ class FetcherSession: def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, + impersonate: Optional[BrowserTypeLiteral] = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -79,986 +1004,71 @@ class FetcherSession: :param cert: Tuple of (cert, key) filenames for the client certificate. :param selector_config: Arguments passed when creating the final Selector class. """ - self.default_impersonate = impersonate - self.stealth = stealthy_headers - self.default_proxies = proxies or {} - self.default_proxy = proxy or None - self.default_proxy_auth = proxy_auth or None - self.default_timeout = timeout - self.default_headers = headers or {} - self.default_retries = retries - self.default_retry_delay = retry_delay - self.default_follow_redirects = follow_redirects - self.default_max_redirects = max_redirects - self.default_verify = verify - self.default_cert = cert - self.default_http3 = http3 + self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate + self._stealth = stealthy_headers + self._default_proxies = proxies or {} + self._default_proxy = proxy or None + self._default_proxy_auth = proxy_auth or None + self._default_timeout = timeout + self._default_headers = headers or {} + self._default_retries = retries + self._default_retry_delay = retry_delay + self._default_follow_redirects = follow_redirects + self._default_max_redirects = max_redirects + self._default_verify = verify + self._default_cert = cert + self._default_http3 = http3 self.selector_config = selector_config or {} + self._client: _SyncSessionLogic | _ASyncSessionLogic | None = None - self._curl_session: Optional[CurlSession] = None - self._async_curl_session: Optional[AsyncCurlSession] = None - - def _merge_request_args(self, **kwargs) -> Dict[str, Any]: - """Merge request-specific arguments with default session arguments.""" - url = kwargs.pop("url") - request_args = {} - - headers = self.get_with_precedence(kwargs, "headers", self.default_headers) - stealth = self.get_with_precedence(kwargs, "stealth", self.stealth) - impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate) - - if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover - request_args["http_version"] = CurlHttpVersion.V3ONLY - if impersonate: - log.warning( - "The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors." - ) - - request_args.update( - { - "url": url, - # Curl automatically generates the suitable browser headers when you use `impersonate` - "headers": self._headers_job(url, headers, stealth, bool(impersonate)), - "proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies), - "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy), - "proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth), - "timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout), - "allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects), - "max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects), - "verify": self.get_with_precedence(kwargs, "verify", self.default_verify), - "cert": self.get_with_precedence(kwargs, "cert", self.default_cert), - "impersonate": impersonate, - **{ - k: v - for k, v in kwargs.items() - if v - not in ( - _UNSET, - None, - ) - }, # Add any remaining parameters (after all known ones are popped) - } - ) - return request_args - - def _headers_job( - self, - url, - headers: Optional[Dict], - stealth: Optional[bool], - impersonate_enabled: bool, - ) -> Dict: - """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and - finally generates a referer header that looks like if this request came from Google's search of the current URL's domain. - - :param headers: Current headers in the request if the user passed any - :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. - :param impersonate_enabled: Whether the browser impersonation is enabled or not. - :return: A dictionary of the new headers. - """ - # Handle headers - if it was _UNSET, use default_headers - if headers is _UNSET: - headers = self.default_headers.copy() - else: - # Merge session headers with request headers, request takes precedence - headers = {**self.default_headers, **(headers or {})} - - headers_keys = set(map(str.lower, headers.keys())) - if stealth: - if "referer" not in headers_keys: - headers.update({"referer": generate_convincing_referer(url)}) - - if impersonate_enabled: # Curl will generate the suitable headers - return headers - - extra_headers = generate_headers(browser_mode=False) - # Don't overwrite user-supplied headers - extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} - headers.update(extra_headers) - - elif "user-agent" not in headers_keys and not impersonate_enabled: - headers["User-Agent"] = __default_useragent__ - log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") - - return headers - - def __enter__(self): + def __enter__(self) -> _SyncSessionLogic: """Creates and returns a new synchronous Fetcher Session""" - if self._curl_session: - raise RuntimeError( - "This FetcherSession instance already has an active synchronous session. " - "Create a new FetcherSession instance for a new independent session, " - "or use the current instance sequentially after the previous context has exited." - ) - if self._async_curl_session: # Prevent mixing if async is active from this instance - raise RuntimeError( - "This FetcherSession instance has an active asynchronous session. " - "Cannot enter a synchronous context simultaneously with the same manager instance." - ) - - self._curl_session = CurlSession() - return self + if self._client is None: + # Use **vars(self) to avoid repeating all parameters + config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config["stealthy_headers"] = self._stealth + config["selector_config"] = self.selector_config + self._client = _SyncSessionLogic(**config) + return self._client.__enter__() + raise RuntimeError("This FetcherSession instance already has an active synchronous session.") def __exit__(self, exc_type, exc_val, exc_tb): - """Closes the active synchronous session managed by this instance, if any.""" - if self._curl_session: - self._curl_session.close() - self._curl_session = None + if self._client is not None and isinstance(self._client, _SyncSessionLogic): + self._client.__exit__(exc_type, exc_val, exc_tb) + self._client = None + return + raise RuntimeError("Cannot exit invalid session") - async def __aenter__(self): + async def __aenter__(self) -> _ASyncSessionLogic: """Creates and returns a new asynchronous Session.""" - if self._async_curl_session: - raise RuntimeError( - "This FetcherSession instance already has an active asynchronous session. " - "Create a new FetcherSession instance for a new independent session, " - "or use the current instance sequentially after the previous context has exited." - ) - if self._curl_session: # Prevent mixing if sync is active from this instance - raise RuntimeError( - "This FetcherSession instance has an active synchronous session. " - "Cannot enter an asynchronous context simultaneously with the same manager instance." - ) - - self._async_curl_session = AsyncCurlSession() - return self + if self._client is None: + # Use **vars(self) to avoid repeating all parameters + config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config["stealthy_headers"] = self._stealth + config["selector_config"] = self.selector_config + self._client = _ASyncSessionLogic(**config) + return await self._client.__aenter__() + raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") async def __aexit__(self, exc_type, exc_val, exc_tb): - """Closes the active asynchronous session managed by this instance, if any.""" - if self._async_curl_session: - await self._async_curl_session.close() - self._async_curl_session = None - - def __make_request( - self, - method: SUPPORTED_HTTP_METHODS, - request_args: Dict[str, Any], - max_retries: int, - retry_delay: int, - selector_config: Dict, - ) -> Response: - """ - Perform an HTTP request using the configured session. - - :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param request_args: Arguments to be passed to the session's `request()` method. - :param max_retries: Maximum number of retries for the request. - :param retry_delay: Number of seconds to wait between retries. - :param selector_config: Arguments passed when creating the final Selector class. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. - """ - session = self._curl_session - if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): - # For usage inside FetcherClient - # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. - session = CurlSession() - - if session: - for attempt in range(max_retries): - try: - response = session.request(method, **request_args) - # response.raise_for_status() # Retry responses with a status code between 200-400 - return ResponseFactory.from_http_request(response, selector_config) - except CurlError as e: # pragma: no cover - if attempt < max_retries - 1: - log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") - time_sleep(retry_delay) - else: - log.error(f"Failed after {max_retries} attempts: {e}") - raise # Raise the exception if all retries fail - - raise RuntimeError("No active session available.") # pragma: no cover - - async def __make_async_request( - self, - method: SUPPORTED_HTTP_METHODS, - request_args: Dict[str, Any], - max_retries: int, - retry_delay: int, - selector_config: Dict, - ) -> Response: - """ - Perform an HTTP request using the configured session. - - :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param request_args: Arguments to be passed to the session's `request()` method. - :param max_retries: Maximum number of retries for the request. - :param retry_delay: Number of seconds to wait between retries. - :param selector_config: Arguments passed when creating the final Selector class. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. - """ - session = self._async_curl_session - if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): - # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons - # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. - # 2. `curl_cffi` doesn't support making async requests without sessions - # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi. - session = AsyncCurlSession() - - if session: - for attempt in range(max_retries): - try: - response = await session.request(method, **request_args) - # response.raise_for_status() # Retry responses with a status code between 200-400 - return ResponseFactory.from_http_request(response, selector_config) - except CurlError as e: # pragma: no cover - if attempt < max_retries - 1: - log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") - await asyncio_sleep(retry_delay) - else: - log.error(f"Failed after {max_retries} attempts: {e}") - raise # Raise the exception if all retries fail - - raise RuntimeError("No active session available.") # pragma: no cover - - @staticmethod - def get_with_precedence(kwargs, key, default_value): - """Get value with request-level priority over session-level""" - request_value = kwargs.pop(key, _UNSET) - return request_value if request_value is not _UNSET else default_value - - def __prepare_and_dispatch( - self, - method: SUPPORTED_HTTP_METHODS, - stealth: Optional[bool] = None, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Internal dispatcher. Prepares arguments and calls sync or async request helper. - - :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. - :param url: Target URL for the request. - :param kwargs: Additional request-specific arguments. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. - """ - stealth = self.stealth if stealth is None else stealth - - selector_config = kwargs.pop("selector_config", {}) or self.selector_config - max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) - retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay) - request_args = self._merge_request_args(stealth=stealth, **kwargs) - if self._curl_session: - return self.__make_request(method, request_args, max_retries, retry_delay, selector_config) - elif self._async_curl_session: - # The returned value is a Coroutine - return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config) - - raise RuntimeError("No active session available.") - - def get( - self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Perform a GET request. - - :param url: Target URL for the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. - """ - request_args = { - "url": url, - "params": params, - "headers": headers, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy": proxy, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "impersonate": impersonate, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args) - - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Perform a POST request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. - """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args) - - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Perform a PUT request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. - """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args) - - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Perform a DELETE request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. - """ - request_args = { - "url": url, - # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, - # But some websites accept it, it depends on the implementation used. - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args) + if self._client is not None and isinstance(self._client, _ASyncSessionLogic): + await self._client.__aexit__(exc_type, exc_val, exc_tb) + self._client = None + return + raise RuntimeError("Cannot exit invalid session") -class FetcherClient(FetcherSession): +class FetcherClient(_SyncSessionLogic): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__enter__: Any = None self.__exit__: Any = None - self.__aenter__: Any = None - self.__aexit__: Any = None - self._curl_session: Any = True - - # Setting the correct return types for the type checking/autocompletion - def get( - self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().get( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().post( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().put( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().delete( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + self._curl_session: Any = _NO_SESSION -class AsyncFetcherClient(FetcherSession): +class AsyncFetcherClient(_ASyncSessionLogic): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.__enter__: Any = None - self.__exit__: Any = None self.__aenter__: Any = None self.__aexit__: Any = None - self._async_curl_session: Any = True - - # Setting the correct return types for the type checking/autocompletion - def get( - self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().get( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().post( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().put( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) - - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().delete( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + self._async_curl_session: Any = _NO_SESSION diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 05dce87..b66b518 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -24,15 +24,15 @@ class ResponseFactory: @classmethod @lru_cache(maxsize=16) - def __extract_browser_encoding(cls, content_type: str | None) -> Optional[str]: + def __extract_browser_encoding(cls, content_type: str | None, default: str = "utf-8") -> str: """Extract browser encoding from headers. Ex: from header "content-type: text/html; charset=utf-8" -> "utf-8 """ if content_type: # Because Playwright can't do that by themselves like all libraries for some reason :3 match = __CHARSET_RE__.search(content_type) - return match.group(1) if match else None - return None + return match.group(1) if match else default + return default @classmethod def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]: @@ -58,7 +58,8 @@ class ResponseFactory: "encoding": cls.__extract_browser_encoding( current_response.headers.get("content-type", "") ) - or "utf-8", + if current_response + else "utf-8", "cookies": tuple(), "headers": current_response.all_headers() if current_response else {}, "request_headers": current_request.all_headers(), @@ -107,15 +108,13 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - encoding = ( - cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" - ) # default encoding + encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) history = cls._process_response_history(first_response, parser_arguments) try: - page_content = page.content() + page_content = final_response.text() except Exception as e: # pragma: no cover log.error(f"Error getting page content: {e}") page_content = "" @@ -161,7 +160,8 @@ class ResponseFactory: "encoding": cls.__extract_browser_encoding( current_response.headers.get("content-type", "") ) - or "utf-8", + if current_response + else "utf-8", "cookies": tuple(), "headers": await current_response.all_headers() if current_response else {}, "request_headers": await current_request.all_headers(), @@ -210,15 +210,13 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - encoding = ( - cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" - ) # default encoding + encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) history = await cls._async_process_response_history(first_response, parser_arguments) try: - page_content = await page.content() + page_content = await final_response.text() except Exception as e: # pragma: no cover log.error(f"Error getting page content in async: {e}") page_content = "" @@ -255,8 +253,8 @@ class ResponseFactory: "encoding": response.encoding or "utf-8", "cookies": dict(response.cookies), "headers": dict(response.headers), - "request_headers": dict(response.request.headers), - "method": response.request.method, + "request_headers": dict(response.request.headers) if response.request else {}, + "method": response.request.method if response.request else "GET", "history": response.history, # https://github.com/lexiforest/curl_cffi/issues/82 **parser_arguments, } diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 774eec7..43ef61c 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -8,6 +8,7 @@ from scrapling.core.utils import log from scrapling.core._types import ( Any, Dict, + cast, List, Optional, Tuple, @@ -30,10 +31,10 @@ class Response(Selector): request_headers: Dict, encoding: str = "utf-8", method: str = "GET", - history: List = None, - **selector_config: Dict, + history: List | None = None, + **selector_config: Any, ): - adaptive_domain = selector_config.pop("adaptive_domain", None) + adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", "")) self.status = status self.reason = reason self.cookies = cookies @@ -58,7 +59,7 @@ class BaseFetcher: keep_cdata: Optional[bool] = False storage_args: Optional[Dict] = None keep_comments: Optional[bool] = False - adaptive_domain: Optional[str] = None + adaptive_domain: str = "" parser_keywords: Tuple = ( "huge_tree", "adaptive", @@ -124,12 +125,8 @@ class BaseFetcher: adaptive=cls.adaptive, storage=cls.storage, storage_args=cls.storage_args, + adaptive_domain=cls.adaptive_domain, ) - if cls.adaptive_domain: - if not isinstance(cls.adaptive_domain, str): - log.warning('[Ignored] The argument "adaptive_domain" must be of string type') - else: - parser_arguments.update({"adaptive_domain": cls.adaptive_domain}) return parser_arguments diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index bd836e7..7bcad8a 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -8,9 +8,10 @@ from platform import system as platform_system from tldextract import extract from browserforge.headers import Browser, HeaderGenerator -from scrapling.core._types import Dict, Optional +from scrapling.core._types import Dict, Literal __OS_NAME__ = platform_system() +OSName = Literal["linux", "macos", "windows"] @lru_cache(10, typed=True) @@ -28,16 +29,20 @@ def generate_convincing_referer(url: str) -> str: @lru_cache(1, typed=True) -def get_os_name() -> Optional[str]: - """Get the current OS name in the same format needed for browserforge +def get_os_name() -> OSName | None: + """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all. :return: Current OS name or `None` otherwise """ - return { - "Linux": "linux", - "Darwin": "macos", - "Windows": "windows", - }.get(__OS_NAME__) + match __OS_NAME__: + case "Linux": + return "linux" + case "Darwin": + return "macos" + case "Windows": + return "windows" + case _: + return None def generate_headers(browser_mode: bool = False) -> Dict: @@ -58,8 +63,10 @@ def generate_headers(browser_mode: bool = False) -> Dict: Browser(name="edge", min_version=130), ] ) - - return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate() + if os_name: + return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate() + else: + return HeaderGenerator(browser=browsers, device="desktop").generate() __default_useragent__ = generate_headers(browser_mode=False).get("User-Agent") diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index ea5991b..959aa5f 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -11,7 +11,7 @@ from msgspec import Struct, structs, convert, ValidationError from playwright.sync_api import Route from scrapling.core.utils import log -from scrapling.core._types import Dict, Optional, Tuple +from scrapling.core._types import Dict, Tuple, overload, Literal from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES __BYPASSES_DIR__ = Path(__file__).parent / "bypasses" @@ -49,7 +49,15 @@ async def async_intercept_route(route: async_Route): await route.continue_() -def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]: +@overload +def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[True]) -> Tuple: ... + + +@overload +def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[False] = False) -> Dict: ... + + +def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: bool = False) -> Dict | Tuple: """Validate a proxy and return it in the acceptable format for Playwright Reference: https://playwright.dev/python/docs/network#http-proxy @@ -83,7 +91,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> except ValidationError as e: raise TypeError(f"Invalid proxy dictionary: {e}") - return None + raise TypeError(f"Invalid proxy string: {proxy_string}") @lru_cache(10, typed=True) diff --git a/scrapling/fetchers/__init__.py b/scrapling/fetchers/__init__.py index 9c64659..c135273 100644 --- a/scrapling/fetchers/__init__.py +++ b/scrapling/fetchers/__init__.py @@ -19,7 +19,17 @@ _LAZY_IMPORTS = { "AsyncStealthySession": ("scrapling.fetchers.firefox", "AsyncStealthySession"), } -__all__ = ["Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"] +__all__ = [ + "Fetcher", + "AsyncFetcher", + "FetcherSession", + "DynamicFetcher", + "DynamicSession", + "AsyncDynamicSession", + "StealthyFetcher", + "StealthySession", + "AsyncStealthySession", +] def __getattr__(name: str) -> Any: diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 49fe009..9cc980d 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -1,10 +1,9 @@ from scrapling.core._types import ( Callable, - Dict, List, + Dict, Optional, SelectorWaitStates, - Iterable, ) from scrapling.engines.toolbelt.custom import BaseFetcher, Response from scrapling.engines._browsers._controllers import DynamicSession, AsyncDynamicSession @@ -47,10 +46,11 @@ class DynamicFetcher(BaseFetcher): disable_resources: bool = False, wait_selector: Optional[str] = None, init_script: Optional[str] = None, - cookies: Optional[Iterable[Dict]] = None, + cookies: Optional[List[Dict]] = None, network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: """Opens up a browser and do your request based on your chosen options below. @@ -80,6 +80,7 @@ class DynamicFetcher(BaseFetcher): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ if not custom_config: @@ -107,6 +108,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, selector_config={**cls._generate_parser_arguments(), **custom_config}, @@ -134,10 +136,11 @@ class DynamicFetcher(BaseFetcher): disable_resources: bool = False, wait_selector: Optional[str] = None, init_script: Optional[str] = None, - cookies: Optional[Iterable[Dict]] = None, + cookies: Optional[List[Dict]] = None, network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: """Opens up a browser and do your request based on your chosen options below. @@ -167,6 +170,7 @@ class DynamicFetcher(BaseFetcher): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ if not custom_config: @@ -195,6 +199,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, selector_config={**cls._generate_parser_arguments(), **custom_config}, diff --git a/scrapling/fetchers/firefox.py b/scrapling/fetchers/firefox.py index ca6db40..5986096 100644 --- a/scrapling/fetchers/firefox.py +++ b/scrapling/fetchers/firefox.py @@ -83,8 +83,6 @@ class StealthyFetcher(BaseFetcher): """ if not custom_config: custom_config = {} - elif not isinstance(custom_config, dict): - ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") with StealthySession( wait=wait, @@ -182,8 +180,6 @@ class StealthyFetcher(BaseFetcher): """ if not custom_config: custom_config = {} - elif not isinstance(custom_config, dict): - ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") async with AsyncStealthySession( wait=wait, diff --git a/scrapling/parser.py b/scrapling/parser.py index 6d0a575..6a4934c 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -17,17 +17,21 @@ from lxml.etree import ( from scrapling.core._types import ( Any, + Set, Dict, + cast, List, Tuple, Union, Pattern, Callable, + Literal, Optional, Iterable, overload, Generator, SupportsIndex, + TYPE_CHECKING, ) from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers from scrapling.core.mixins import SelectorsGeneration @@ -36,7 +40,7 @@ from scrapling.core.storage import ( StorageSystemMixin, _StorageTools, ) -from scrapling.core.translator import translator as _translator +from scrapling.core.translator import css_to_xpath as _css_to_xpath from scrapling.core.utils import clean_spaces, flatten, html_forbidden, log __DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db") @@ -70,20 +74,23 @@ class Selector(SelectorsGeneration): "_raw_body", ) + if TYPE_CHECKING: + _storage: StorageSystemMixin + def __init__( self, content: Optional[str | bytes] = None, - url: Optional[str] = None, + url: str = "", encoding: str = "utf-8", huge_tree: bool = True, root: Optional[HtmlElement] = None, keep_comments: Optional[bool] = False, keep_cdata: Optional[bool] = False, adaptive: Optional[bool] = False, - _storage: object = None, + _storage: Optional[StorageSystemMixin] = None, storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, - **kwargs, + **_, ): """The main class that works as a wrapper for the HTML input data. Using this class, you can search for elements with expressions in CSS, XPath, or with simply text. Check the docs for more info. @@ -131,7 +138,7 @@ class Selector(SelectorsGeneration): default_doctype=True, strip_cdata=(not keep_cdata), ) - self._root = fromstring(body, parser=parser, base_url=url) + self._root = cast(HtmlElement, fromstring(body, parser=parser, base_url=url or None)) self._raw_body = content else: @@ -141,7 +148,7 @@ class Selector(SelectorsGeneration): f"Root have to be a valid element of `html` module types to work, not of type {type(root)}" ) - self._root = root + self._root = cast(HtmlElement, root) self._raw_body = "" self.__adaptive_enabled = adaptive @@ -238,6 +245,9 @@ class Selector(SelectorsGeneration): **self.__response_data, ) + def __elements_convertor(self, elements: List[HtmlElement]) -> "Selectors": + return Selectors(map(self.__element_convertor, elements)) + def __handle_element( self, element: Optional[HtmlElement | _ElementUnicodeResult] ) -> Optional[Union[TextHandler, "Selector"]]: @@ -262,7 +272,7 @@ class Selector(SelectorsGeneration): if self._is_text_node(result[0]): return TextHandlers(map(TextHandler, result)) - return Selectors(map(self.__element_convertor, result)) + return self.__elements_convertor(result) def __getstate__(self) -> Any: # lxml don't like it :) @@ -323,7 +333,7 @@ class Selector(SelectorsGeneration): if not valid_values or processed_text.strip(): _all_strings.append(processed_text) - return TextHandler(separator).join(_all_strings) + return cast(TextHandler, TextHandler(separator).join(_all_strings)) def urljoin(self, relative_url: str) -> str: """Join this Selector's url with a relative url to form an absolute full URL.""" @@ -372,13 +382,14 @@ class Selector(SelectorsGeneration): @property def parent(self) -> Optional["Selector"]: """Return the direct parent of the element or ``None`` otherwise""" - return self.__handle_element(self._root.getparent()) + _parent = self._root.getparent() + return self.__element_convertor(_parent) if _parent is not None else None @property def below_elements(self) -> "Selectors": """Return all elements under the current element in the DOM tree""" below = _find_all_elements(self._root) - return self.__handle_elements(below) + return self.__elements_convertor(below) if below is not None else Selectors() @property def children(self) -> "Selectors": @@ -425,7 +436,7 @@ class Selector(SelectorsGeneration): # Ignore HTML comments and unwanted types next_element = next_element.getnext() - return self.__handle_element(next_element) + return self.__element_convertor(next_element) if next_element is not None else None @property def previous(self) -> Optional["Selector"]: @@ -435,10 +446,10 @@ class Selector(SelectorsGeneration): # Ignore HTML comments and unwanted types prev_element = prev_element.getprevious() - return self.__handle_element(prev_element) + return self.__element_convertor(prev_element) if prev_element is not None else None # For easy copy-paste from Scrapy/parsel code when needed :) - def get(self, default=None): + def get(self, default=None): # pyright: ignore return self def get_all(self): @@ -468,6 +479,16 @@ class Selector(SelectorsGeneration): return data + ">" # From here we start with the selecting functions + @overload + def relocate( + self, element: Union[Dict, HtmlElement, "Selector"], percentage: int, selector_type: Literal[True] + ) -> "Selectors": ... + + @overload + def relocate( + self, element: Union[Dict, HtmlElement, "Selector"], percentage: int, selector_type: Literal[False] = False + ) -> List[HtmlElement]: ... + def relocate( self, element: Union[Dict, HtmlElement, "Selector"], @@ -506,11 +527,11 @@ class Selector(SelectorsGeneration): log.debug(f"Highest probability was {highest_probability}%") log.debug("Top 5 best matching elements are: ") for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]: - log.debug(f"{percent} -> {self.__handle_elements(score_table[percent])}") + log.debug(f"{percent} -> {self.__elements_convertor(score_table[percent])}") if not selector_type: return score_table[highest_probability] - return self.__handle_elements(score_table[highest_probability]) + return self.__elements_convertor(score_table[highest_probability]) return [] def css_first( @@ -593,7 +614,7 @@ class Selector(SelectorsGeneration): auto_save: bool = False, percentage: int = 0, **kwargs: Any, - ) -> Union["Selectors", List, "TextHandlers"]: + ) -> Union["Selectors", List[Any], "TextHandlers"]: """Search the current tree with CSS3 selectors **Important: @@ -614,7 +635,7 @@ class Selector(SelectorsGeneration): try: if not self.__adaptive_enabled or "," not in selector: # No need to split selectors in this case, let's save some CPU cycles :) - xpath_selector = _translator.css_to_xpath(selector) + xpath_selector = _css_to_xpath(selector) return self.xpath( xpath_selector, identifier or selector, @@ -628,7 +649,7 @@ class Selector(SelectorsGeneration): for single_selector in split_selectors(selector): # I'm doing this only so the `save` function saves data correctly for combined selectors # Like using the ',' to combine two different selectors that point to different elements. - xpath_selector = _translator.css_to_xpath(single_selector.canonical()) + xpath_selector = _css_to_xpath(single_selector.canonical()) results += self.xpath( xpath_selector, identifier or single_selector.canonical(), @@ -731,7 +752,8 @@ class Selector(SelectorsGeneration): raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.") attributes = dict() - tags, patterns = set(), set() + tags: Set[str] = set() + patterns: Set[Pattern] = set() results, functions, selectors = Selectors(), [], [] # Brace yourself for a wonderful journey! @@ -740,6 +762,7 @@ class Selector(SelectorsGeneration): tags.add(arg) elif type(arg) in (list, tuple, set): + arg = cast(Iterable, arg) # Type narrowing for type checkers like pyright if not all(map(lambda x: isinstance(x, str), arg)): raise TypeError("Nested Iterables are not accepted, only iterables of tag names are accepted") tags.update(set(arg)) @@ -774,7 +797,7 @@ class Selector(SelectorsGeneration): attributes[attribute_name] = value # It's easier and faster to build a selector than traversing the tree - tags = tags or ["*"] + tags = tags or set("*") for tag in tags: selector = tag for key, value in attributes.items(): @@ -785,7 +808,7 @@ class Selector(SelectorsGeneration): selectors.append(selector) if selectors: - results = self.css(", ".join(selectors)) + results = cast(Selectors, self.css(", ".join(selectors))) if results: # From the results, get the ones that fulfill passed regex patterns for pattern in patterns: @@ -828,20 +851,20 @@ class Selector(SelectorsGeneration): :return: A percentage score of how similar is the candidate to the original element """ score, checks = 0, 0 - candidate = _StorageTools.element_to_dict(candidate) + data = _StorageTools.element_to_dict(candidate) # Possible TODO: # Study the idea of giving weight to each test below so some are more important than others # Current results: With weights some websites had better score while it was worse for others - score += 1 if original["tag"] == candidate["tag"] else 0 # * 0.3 # 30% + score += 1 if original["tag"] == data["tag"] else 0 # * 0.3 # 30% checks += 1 if original["text"]: - score += SequenceMatcher(None, original["text"], candidate.get("text") or "").ratio() # * 0.3 # 30% + score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio() # * 0.3 # 30% checks += 1 # if both don't have attributes, it still counts for something! - score += self.__calculate_dict_diff(original["attributes"], candidate["attributes"]) # * 0.3 # 30% + score += self.__calculate_dict_diff(original["attributes"], data["attributes"]) # * 0.3 # 30% checks += 1 # Separate similarity test for class, id, href,... this will help in full structural changes @@ -855,23 +878,23 @@ class Selector(SelectorsGeneration): score += SequenceMatcher( None, original["attributes"][attrib], - candidate["attributes"].get(attrib) or "", + data["attributes"].get(attrib) or "", ).ratio() # * 0.3 # 30% checks += 1 - score += SequenceMatcher(None, original["path"], candidate["path"]).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["path"], data["path"]).ratio() # * 0.1 # 10% checks += 1 if original.get("parent_name"): # Then we start comparing parents' data - if candidate.get("parent_name"): + if data.get("parent_name"): score += SequenceMatcher( - None, original["parent_name"], candidate.get("parent_name") or "" + None, original["parent_name"], data.get("parent_name") or "" ).ratio() # * 0.2 # 20% checks += 1 score += self.__calculate_dict_diff( - original["parent_attribs"], candidate.get("parent_attribs") or {} + original["parent_attribs"], data.get("parent_attribs") or {} ) # * 0.2 # 20% checks += 1 @@ -879,7 +902,7 @@ class Selector(SelectorsGeneration): score += SequenceMatcher( None, original["parent_text"], - candidate.get("parent_text") or "", + data.get("parent_text") or "", ).ratio() # * 0.1 # 10% checks += 1 # else: @@ -887,9 +910,7 @@ class Selector(SelectorsGeneration): # score -= 0.1 if original.get("siblings"): - score += SequenceMatcher( - None, original["siblings"], candidate.get("siblings") or [] - ).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio() # * 0.1 # 10% checks += 1 # How % sure? let's see @@ -902,7 +923,7 @@ class Selector(SelectorsGeneration): score += SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() * 0.5 return score - def save(self, element: Union["Selector", HtmlElement], identifier: str) -> None: + def save(self, element: HtmlElement, identifier: str) -> None: """Saves the element's unique properties to the storage for retrieval and relocation later :param element: The element itself that we want to save to storage, it can be a ` Selector ` or pure ` HtmlElement ` @@ -910,15 +931,16 @@ class Selector(SelectorsGeneration): the docs for more info. """ if self.__adaptive_enabled: - if isinstance(element, self.__class__): - element = element._root + target = element + if isinstance(target, self.__class__): + target: HtmlElement = target._root - if self._is_text_node(element): - element = element.getparent() + if self._is_text_node(target): + target: HtmlElement = target.getparent() - self._storage.save(element, identifier) + self._storage.save(target, identifier) else: - log.critical( + raise RuntimeError( "Can't use `adaptive` features while it's disabled globally, you have to start a new class instance." ) @@ -932,10 +954,9 @@ class Selector(SelectorsGeneration): if self.__adaptive_enabled: return self._storage.retrieve(identifier) - log.critical( + raise RuntimeError( "Can't use `adaptive` features while it's disabled globally, you have to start a new class instance." ) - return None # Operations on text functions def json(self) -> Dict: @@ -1104,28 +1125,30 @@ class Selector(SelectorsGeneration): if not case_sensitive: text = text.lower() - for node in self.__handle_elements(_find_all_elements_with_spaces(self._root)): - """Check if element matches given text otherwise, traverse the children tree and iterate""" - node_text = node.text - if clean_match: - node_text = node_text.clean() + possible_targets = _find_all_elements_with_spaces(self._root) + if possible_targets: + for node in self.__elements_convertor(possible_targets): + """Check if element matches given text otherwise, traverse the children tree and iterate""" + node_text = node.text + if clean_match: + node_text = node_text.clean() - if not case_sensitive: - node_text = node_text.lower() + if not case_sensitive: + node_text = node_text.lower() - if partial: - if text in node_text: + if partial: + if text in node_text: + results.append(node) + elif text == node_text: results.append(node) - elif text == node_text: - results.append(node) - if first_match and results: - # we got an element so we should stop - break + if first_match and results: + # we got an element so we should stop + break - if first_match: - if results: - return results[0] + if first_match: + if results: + return results[0] return results def find_by_regex( @@ -1143,23 +1166,25 @@ class Selector(SelectorsGeneration): """ results = Selectors() - for node in self.__handle_elements(_find_all_elements_with_spaces(self._root)): - """Check if element matches given regex otherwise, traverse the children tree and iterate""" - node_text = node.text - if node_text.re( - query, - check_match=True, - clean_match=clean_match, - case_sensitive=case_sensitive, - ): - results.append(node) + possible_targets = _find_all_elements_with_spaces(self._root) + if possible_targets: + for node in self.__elements_convertor(possible_targets): + """Check if element matches given regex otherwise, traverse the children tree and iterate""" + node_text = node.text + if node_text.re( + query, + check_match=True, + clean_match=clean_match, + case_sensitive=case_sensitive, + ): + results.append(node) - if first_match and results: - # we got an element so we should stop - break + if first_match and results: + # we got an element so we should stop + break - if results and first_match: - return results[0] + if results and first_match: + return results[0] return results @@ -1181,9 +1206,9 @@ class Selectors(List[Selector]): def __getitem__(self, pos: SupportsIndex | slice) -> Union[Selector, "Selectors"]: lst = super().__getitem__(pos) if isinstance(pos, slice): - return self.__class__(lst) + return self.__class__(cast(List[Selector], lst)) else: - return lst + return cast(Selector, lst) def xpath( self, @@ -1265,7 +1290,7 @@ class Selectors(List[Selector]): def re_first( self, regex: str | Pattern, - default=None, + default: Any = None, replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, diff --git a/setup.cfg b/setup.cfg index 6b0e5a1..cd31254 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.6 +version = 0.3.7 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.py index c9abc9c..c4e1355 100644 --- a/tests/fetchers/async/test_requests_session.py +++ b/tests/fetchers/async/test_requests_session.py @@ -14,4 +14,3 @@ class TestFetcherSession: # Should not have context manager methods assert client.__aenter__ is None assert client.__aexit__ is None - assert client._async_curl_session is True # Special marker diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.py index 8b7905e..152fbc4 100644 --- a/tests/fetchers/sync/test_requests_session.py +++ b/tests/fetchers/sync/test_requests_session.py @@ -1,7 +1,7 @@ import pytest -from scrapling.engines.static import FetcherSession, FetcherClient +from scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient class TestFetcherSession: @@ -15,9 +15,8 @@ class TestFetcherSession: stealthy_headers=True ) - assert session.default_timeout == 30 - assert session.default_retries == 3 - assert session.stealth is True + assert session._default_timeout == 30 + assert session._default_retries == 3 def test_fetcher_session_context_manager(self): """Test FetcherSession as a context manager""" @@ -44,4 +43,3 @@ class TestFetcherSession: # Should not have context manager methods assert client.__enter__ is None assert client.__exit__ is None - assert client._curl_session is True # Special marker