diff --git a/.github/ISSUE_TEMPLATE/03-other.yml b/.github/ISSUE_TEMPLATE/03-other.yml new file mode 100644 index 0000000..697c1f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/03-other.yml @@ -0,0 +1,19 @@ +name: Other +description: Use this for any other issues. PLEASE provide as much information as possible. +labels: ["awaiting triage"] +body: + - type: textarea + id: issuedescription + attributes: + label: What would you like to share? + description: Provide a clear and concise explanation of your issue. + validations: + required: true + + - type: textarea + id: extrainfo + attributes: + label: Additional information + description: Is there anything else we should know about this issue? + validations: + required: false \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 88737fa..10a6740 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,15 @@ on: branches: - main - dev + paths-ignore: + - '*.md' + - '**/*.md' + - 'docs/*' + - 'images/*' + - '.github/*' + - '*.yml' + - '*.yaml' + - 'ruff.toml' concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e4885b..a67bd50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.8.3 + rev: 1.8.6 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.11.5 + rev: v0.13.0 hooks: # Run the linter. - id: ruff diff --git a/README.md b/README.md index 912350e..f344704 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet - 🔄 **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms. - 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more. - 🔍 **Find Similar Elements**: Automatically locate elements similar to found elements. -- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage. +- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage. ([demo video](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) ### High-Performance & battle-tested Architecture - 🚀 **Lightning Fast**: Optimized performance outperforming most Python scraping libraries. @@ -134,7 +134,7 @@ quotes = page.css('.quote .text::text') # Advanced stealth mode (Keep the browser open until you finish) with StealthySession(headless=True, solve_cloudflare=True) as session: - page = session.fetch('https://nopecha.com/demo/cloudflare') + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) data = page.css('#padded_content a') # Or use one-off request style, it opens the browser for this request, then closes it after finishing @@ -143,7 +143,7 @@ data = page.css('#padded_content a') # Full browser automation (Keep the browser open until you finish) with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session: - page = session.fetch('https://quotes.toscrape.com/') + page = session.fetch('https://quotes.toscrape.com/', load_dom=False) data = page.xpath('//span[@class="text"]/text()') # XPath selector if you prefer it # Or use one-off request style, it opens the browser for this request, then closes it after finishing @@ -187,7 +187,7 @@ from scrapling.parser import Selector page = Selector("...") ``` -And it works exactly the same way! +And it works precisely the same way! ### Async Session Management Examples ```python @@ -236,20 +236,20 @@ scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas. ## Performance Benchmarks -Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations! +Scrapling isn't just powerful—it's also blazing fast, and the updates since version 0.3 deliver exceptional performance improvements across all operations! ### Text Extraction Speed Test (5000 nested elements) | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.88 | 1.0x | -| 2 | Parsel/Scrapy | 1.96 | 1.043x | -| 3 | Raw Lxml | 2.32 | 1.234x | -| 4 | PyQuery | 20.2 | ~11x | -| 5 | Selectolax | 85.2 | ~45x | -| 6 | MechanicalSoup | 1305.84 | ~695x | -| 7 | BS4 with Lxml | 1307.92 | ~696x | -| 8 | BS4 with html5lib | 3336.28 | ~1775x | +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | ### Element Similarity & Text Search Performance @@ -257,8 +257,8 @@ Scrapling's adaptive element finding capabilities significantly outperform alter | Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 2.02 | 1.0x | -| AutoScraper | 10.26 | 5.08x | +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | > All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology. @@ -271,29 +271,33 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -#### Fetchers Setup - -If you are going to use any of the fetchers or their classes, then install browser dependencies with -```bash -scrapling install -``` - -This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies. ### Optional Dependencies -- Install the MCP server feature: -```bash -pip install "scrapling[ai]" -``` -- Install shell features (Web Scraping shell and the `extract` command): -```bash -pip install "scrapling[shell]" -``` -- Install everything: -```bash -pip install "scrapling[all]" -``` +1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies, and then install their browser dependencies with + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. + +2. Extra features: + - Install the MCP server feature: + ```bash + pip install "scrapling[ai]" + ``` + - Install shell features (Web Scraping shell and the `extract` command): + ```bash + pip install "scrapling[shell]" + ``` + - Install everything: + ```bash + pip install "scrapling[all]" + ``` + Don't forget that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) ## Contributing @@ -322,4 +326,4 @@ This project includes code adapted from: - [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches) for stealth improvements --- -
Designed & crafted with ❤️ by Karim Shoair.

+
Designed & crafted with ❤️ by Karim Shoair.

\ No newline at end of file diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md index ceef23a..19088ec 100644 --- a/docs/ai/mcp-server.md +++ b/docs/ai/mcp-server.md @@ -1,5 +1,7 @@ # Scrapling MCP Server Guide + + The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful Web Scraping capabilities directly to your favorite AI chatbot or AI agent. This integration allows you to scrape websites, extract data, and bypass anti-bot protections conversationally through Claude's AI interface or any other chatbot that supports MCP. ## Features diff --git a/docs/benchmarks.md b/docs/benchmarks.md index ceb35e4..4e207ad 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,6 +1,6 @@ # Performance Benchmarks -Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations! +Scrapling isn't just powerful—it's also blazing fast, and the updates since version 0.3 deliver exceptional performance improvements across all operations! ## Benchmark Results @@ -8,14 +8,14 @@ Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.88 | 1.0x | -| 2 | Parsel/Scrapy | 1.96 | 1.043x | -| 3 | Raw Lxml | 2.32 | 1.234x | -| 4 | PyQuery | 20.2 | ~11x | -| 5 | Selectolax | 85.2 | ~45x | -| 6 | MechanicalSoup | 1305.84 | ~695x | -| 7 | BS4 with Lxml | 1307.92 | ~696x | -| 8 | BS4 with html5lib | 3336.28 | ~1775x | +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | ### Element Similarity & Text Search Performance @@ -23,5 +23,5 @@ Scrapling's adaptive element finding capabilities significantly outperform alter | Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 2.02 | 1.0x | -| AutoScraper | 10.26 | 5.08x | +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index e6566c5..11b5e41 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -14,10 +14,7 @@ Check out how to configure the parsing options [here](choosing.md#parser-configu Now, we will review most of the arguments one by one, using examples. If you want to jump to a table of all arguments for quick reference, [click here](#full-list-of-arguments) -> Notes: -> -> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state). -> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. +> Note: The async version of the `fetch` method is the `async_fetch` method, of course. This fetcher currently provides four main run options, which can be mixed as desired. @@ -65,7 +62,7 @@ DynamicFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222') Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). ## Full list of arguments -Scrapling provides many options with this fetcher. To make it as simple as possible, we will list the options here and give examples of using most of them. +Scrapling provides many options with this fetcher and its session classes. To make it as simple as possible, we will list the options here and give examples of using most of them. | Argument | Description | Optional | |:-------------------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| @@ -75,9 +72,10 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | cookies | Set cookies for the next request. | ✔️ | | useragent | Pass a useragent string to be used. **Otherwise, the fetcher will generate and use a real Useragent of the same browser.** | ✔️ | | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ | | timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | -| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | +| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | | init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | @@ -92,6 +90,9 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | +In the session classes, all these arguments can be set for the session globally. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, and `selector_config`. + + ## Examples It's easier to understand with examples, so let's take a look. @@ -134,7 +135,6 @@ def scroll_page(page: Page): page.mouse.wheel(10, 0) page.mouse.move(100, 400) page.mouse.up() - return page page = DynamicFetcher.fetch( 'https://example.com', @@ -149,7 +149,6 @@ async def scroll_page(page: Page): await page.mouse.wheel(10, 0) await page.mouse.move(100, 400) await page.mouse.up() - return page page = await DynamicFetcher.async_fetch( 'https://example.com', @@ -169,7 +168,7 @@ page = DynamicFetcher.fetch( ``` This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. -After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. +After that, if `load_dom` is enabled (the default), the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): @@ -273,9 +272,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: -When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. + +This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) + +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws since it's nearly impossible to protect pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 667f43e..d998246 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -12,13 +12,10 @@ You have one primary way to import this Fetcher, which is the same for all fetch ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) -> Notes: -> -> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state). -> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. +> Note: The async version of the `fetch` method is the `async_fetch` method, of course. ## Full list of arguments -Before jumping to [examples](#examples), here's the full list of arguments +Scrapling provides many options with this fetcher and its session classes. Before jumping to the [examples](#examples), here's the full list of arguments | Argument | Description | Optional | @@ -31,7 +28,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ | | block_webrtc | Blocks WebRTC entirely. | ✔️ | -| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | +| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ | | addons | List of Firefox addons to use. **Must be paths to extracted addons.** | ✔️ | | humanize | Humanize the cursor movement. The cursor movement takes either True or the maximum duration in seconds. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ | | allow_webgl | Enabled by default. Disabling WebGL is not recommended, as many WAFs now check if WebGL is enabled. | ✔️ | @@ -40,6 +37,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | disable_ads | Disabled by default; this installs the `uBlock Origin` addon on the browser if enabled. | ✔️ | | solve_cloudflare | When enabled, fetcher solves all three types of Cloudflare's Turnstile wait/captcha page before returning the response to you. | ✔️ | | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ | | timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | @@ -49,6 +47,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | additional_args | Additional arguments to be passed to Camoufox as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | +In the session classes, all these arguments can be set for the session globally. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, and `selector_config`. ## Examples It's easier to understand with examples, so we will now review most of the arguments individually with examples. @@ -156,7 +155,6 @@ def scroll_page(page: Page): page.mouse.wheel(10, 0) page.mouse.move(100, 400) page.mouse.up() - return page page = StealthyFetcher.fetch( 'https://example.com', @@ -171,7 +169,6 @@ async def scroll_page(page: Page): await page.mouse.wheel(10, 0) await page.mouse.move(100, 400) await page.mouse.up() - return page page = await StealthyFetcher.async_fetch( 'https://example.com', @@ -190,7 +187,7 @@ page = StealthyFetcher.fetch( ``` This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. -After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. +After that, if `load_dom` is enabled (the default), the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): @@ -278,9 +275,14 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of using one tab for all your requests, you set a limit on the maximum number of pages allowed. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: -When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason. +1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive for some reason. + +This logic allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) + +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved to have flaws since it's nearly impossible to protect pages/tabs from contamination of the previous configuration you used with the request before this one. ### Session Benefits diff --git a/docs/index.md b/docs/index.md index 75accc9..51d490d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -114,29 +114,33 @@ Scrapling requires Python 3.10 or higher: pip install scrapling ``` -#### Fetchers Setup - -If you are going to use any of the fetchers or their session classes, then install browser dependencies with -```bash -scrapling install -``` - -This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. +Starting with v0.3.2, this installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies. ### Optional Dependencies -- Install the MCP server feature: -```bash -pip install "scrapling[ai]" -``` -- Install shell features (Web Scraping shell and the `extract` command): -```bash -pip install "scrapling[shell]" -``` -- Install everything: -```bash -pip install "scrapling[all]" -``` +1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies, and then install their browser dependencies with + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + This downloads all browsers with their system dependencies and fingerprint manipulation dependencies. + +2. Extra features: + - Install the MCP server feature: + ```bash + pip install "scrapling[ai]" + ``` + - Install shell features (Web Scraping shell and the `extract` command): + ```bash + pip install "scrapling[shell]" + ``` + - Install everything: + ```bash + pip install "scrapling[all]" + ``` + Don't forget that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) ## How the documentation is organized Scrapling has a lot of documentation, so we try to follow a guideline called the [Diátaxis documentation framework](https://diataxis.fr/). diff --git a/docs/overview.md b/docs/overview.md index acde1b3..561e10b 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -292,7 +292,6 @@ It's built on top of [Playwright](https://playwright.dev/python/) and it's curre - Stealthy Playwright with custom stealth mode explicitly written for it. It's not top-tier stealth mode, but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/). Check out the `StealthyFetcher` class below for more advanced stealth mode. It uses the Chromium browser. - Real browsers like your Chrome browser by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it. -> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later. Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments. @@ -314,7 +313,6 @@ True >>> page.status == 200 True ``` -> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later. Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments. diff --git a/pyproject.toml b/pyproject.toml index ad4550f..53f5a4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,15 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "lxml>=6.0.0", + "lxml>=6.0.1", "cssselect>=1.3.0", - "click>=8.2.1", - "orjson>=3.11.2", + "orjson>=3.11.3", "tldextract>=5.3.0", +] + +[project.optional-dependencies] +fetchers = [ + "click>=8.2.1", "curl_cffi>=0.13.0", "playwright>=1.52.0", "rebrowser-playwright>=1.52.0", @@ -68,15 +72,15 @@ dependencies = [ "geoip2>=5.1.0", "msgspec>=0.19.0", ] - -[project.optional-dependencies] ai = [ - "mcp>=1.13.0", + "mcp>=1.14.0", "markdownify>=1.2.0", + "scrapling[fetchers]", ] shell = [ "IPython>=8.37", # The last version that supports Python 3.10 "markdownify>=1.2.0", + "scrapling[fetchers]", ] all = [ "scrapling[ai,shell]", diff --git a/ruff.toml b/ruff.toml index a579697..405614a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -10,8 +10,10 @@ exclude = [ "benchmarks.py", ] -# Assume Python 3.9 -target-version = "py39" +# Assume Python 3.10 +target-version = "py310" +# Allow lines to be as long as 120. +line-length = 120 [lint] select = ["E", "F", "W"] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index f08e652..d2b54ce 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.1" +__version__ = "0.3.2" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/scrapling/cli.py b/scrapling/cli.py index a914ffe..48e99bf 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,14 +2,18 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable -from scrapling.core.utils import log -from scrapling.engines.toolbelt import Response +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _CookieParser, _ParseHeaders from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable -from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher -from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders from orjson import loads as json_loads, JSONDecodeError -from click import command, option, Choice, group, argument + +try: + from click import command, option, Choice, group, argument +except (ImportError, ModuleNotFoundError) as e: + raise ModuleNotFoundError( + "You need to install scrapling with any of the extras to enable Shell commands. See: https://scrapling.readthedocs.io/en/latest/#installation" + ) from e __OUTPUT_FILE_HELP__ = "The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively." __PACKAGE_DIR__ = Path(__file__).parent @@ -40,6 +44,8 @@ def __Request_and_Save( **kwargs, ) -> None: """Make a request using the specified fetcher function and save the result""" + from scrapling.core.shell import Convertor + # Handle relative paths - convert to an absolute path based on the current working directory output_path = Path(output_file) if not output_path.is_absolute(): @@ -72,14 +78,10 @@ def __ParseExtractArguments( return parsed_headers, parsed_cookies, parsed_params, parsed_json -def __BuildRequest( - headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs -) -> Dict: +def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs) -> Dict: """Build a request object using the specified arguments""" # Parse parameters - parsed_headers, parsed_cookies, parsed_params, parsed_json = ( - __ParseExtractArguments(headers, cookies, params, json) - ) + parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json) # Build request arguments request_kwargs = { "headers": parsed_headers if parsed_headers else None, @@ -106,10 +108,7 @@ def __BuildRequest( help="Force Scrapling to reinstall all Fetchers dependencies", ) def install(force): # pragma: no cover - if ( - force - or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists() - ): + if force or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists(): __Execute( [python_executable, "-m", "playwright", "install", "chromium"], "Playwright browsers", @@ -158,9 +157,7 @@ def mcp(): "level", is_flag=False, default="debug", - type=Choice( - ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False - ), + type=Choice(["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False), help="Log level (default: DEBUG)", ) def shell(code, level): @@ -178,9 +175,7 @@ def extract(): pass -@extract.command( - help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -190,9 +185,7 @@ def extract(): help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -264,12 +257,12 @@ def get( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -285,9 +278,7 @@ def get( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -364,12 +355,12 @@ def post( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option("--data", "-d", help="Form data to include in the request body") @@ -381,9 +372,7 @@ def post( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -460,12 +449,12 @@ def put( proxy=proxy, data=data, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -475,9 +464,7 @@ def put( help='HTTP headers in format "Key: Value" (can be used multiple times)', ) @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option( - "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" -) +@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--css-selector", @@ -549,12 +536,12 @@ def delete( impersonate=impersonate, proxy=proxy, ) + from scrapling.fetchers import Fetcher + __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -591,9 +578,7 @@ def delete( ) @option("--wait-selector", help="CSS selector to wait for before proceeding") @option("--locale", default="en-US", help="Browser locale (default: en-US)") -@option( - "--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)" -) +@option("--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)") @option( "--hide-canvas/--show-canvas", default=False, @@ -672,12 +657,12 @@ def fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import DynamicFetcher + __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) -@extract.command( - help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}" -) +@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) @option( @@ -821,6 +806,8 @@ def stealthy_fetch( if parsed_headers: kwargs["extra_headers"] = parsed_headers + from scrapling.fetchers import StealthyFetcher + __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs) diff --git a/scrapling/core/_html_utils.py b/scrapling/core/_html_utils.py index 99776af..6b09830 100644 --- a/scrapling/core/_html_utils.py +++ b/scrapling/core/_html_utils.py @@ -269,17 +269,13 @@ name2codepoint = { } -def to_unicode( - text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict" -) -> str: +def to_unicode(text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict") -> str: """Return the Unicode representation of a bytes object `text`. If `text` is already a Unicode object, return it as-is.""" if isinstance(text, str): return text if not isinstance(text, (bytes, str)): - raise TypeError( - f"to_unicode must receive bytes or str, got {type(text).__name__}" - ) + raise TypeError(f"to_unicode must receive bytes or str, got {type(text).__name__}") if encoding is None: encoding = "utf-8" return text.decode(encoding, errors) @@ -328,9 +324,7 @@ def _replace_entities( entity_name = groups["named"] if entity_name.lower() in keep: return m.group(0) - number = name2codepoint.get(entity_name) or name2codepoint.get( - entity_name.lower() - ) + number = name2codepoint.get(entity_name) or name2codepoint.get(entity_name.lower()) if number is not None: # Browsers typically # interpret numeric character references in the 80-9F range as representing the characters mapped diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index aa2d52b..ae517fa 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -4,7 +4,7 @@ from mcp.server.fastmcp import FastMCP from pydantic import BaseModel, Field from scrapling.core.shell import Convertor -from scrapling.engines.toolbelt import Response as _ScraplingResponse +from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse from scrapling.fetchers import ( Fetcher, FetcherSession, @@ -32,21 +32,13 @@ class ResponseModel(BaseModel): """Request's response information structure.""" status: int = Field(description="The status code returned by the website.") - content: list[str] = Field( - description="The content as Markdown/HTML or the text content of the page." - ) - url: str = Field( - description="The URL given by the user that resulted in this response." - ) + content: list[str] = Field(description="The content as Markdown/HTML or the text content of the page.") + url: str = Field(description="The URL given by the user that resulted in this response.") -def _ContentTranslator( - content: Generator[str, None, None], page: _ScraplingResponse -) -> ResponseModel: +def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: """Convert a content generator to a list of ResponseModel objects.""" - return ResponseModel( - status=page.status, content=[result for result in content], url=page.url - ) + return ResponseModel(status=page.status, content=[result for result in content], url=page.url) class ScraplingMCPServer: diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index ef76880..eb7fa34 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -31,15 +31,11 @@ class TextHandler(str): __slots__ = () - def __getitem__( - self, key: SupportsIndex | slice - ) -> "TextHandler": # pragma: no cover + def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": # pragma: no cover lst = super().__getitem__(key) return cast(_TextHandlerType, TextHandler(lst)) - def split( - self, sep: str = None, maxsplit: SupportsIndex = -1 - ) -> "TextHandlers": # pragma: no cover + def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": # pragma: no cover return TextHandlers( cast( List[_TextHandlerType], @@ -50,14 +46,10 @@ class TextHandler(str): def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().strip(chars)) - def lstrip( - self, chars: str = None - ) -> Union[str, "TextHandler"]: # pragma: no cover + def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().lstrip(chars)) - def rstrip( - self, chars: str = None - ) -> Union[str, "TextHandler"]: # pragma: no cover + def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().rstrip(chars)) def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover @@ -66,37 +58,25 @@ class TextHandler(str): def casefold(self) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().casefold()) - def center( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def center(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().center(width, fillchar)) - def expandtabs( - self, tabsize: SupportsIndex = 8 - ) -> Union[str, "TextHandler"]: # pragma: no cover + def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().expandtabs(tabsize)) - def format( - self, *args: str, **kwargs: str - ) -> Union[str, "TextHandler"]: # pragma: no cover + def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format(*args, **kwargs)) def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format_map(mapping)) - def join( - self, iterable: Iterable[str] - ) -> Union[str, "TextHandler"]: # pragma: no cover + def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().join(iterable)) - def ljust( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def ljust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().ljust(width, fillchar)) - def rjust( - self, width: SupportsIndex, fillchar: str = " " - ) -> Union[str, "TextHandler"]: # pragma: no cover + def rjust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().rjust(width, fillchar)) def swapcase(self) -> Union[str, "TextHandler"]: # pragma: no cover @@ -108,14 +88,10 @@ class TextHandler(str): def translate(self, table) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().translate(table)) - def zfill( - self, width: SupportsIndex - ) -> Union[str, "TextHandler"]: # pragma: no cover + def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().zfill(width)) - def replace( - self, old: str, new: str, count: SupportsIndex = -1 - ) -> Union[str, "TextHandler"]: + def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, "TextHandler"]: return TextHandler(super().replace(old, new, count)) def upper(self) -> Union[str, "TextHandler"]: @@ -203,11 +179,7 @@ class TextHandler(str): results = flatten(results) if not replace_entities: - return TextHandlers( - cast( - List[_TextHandlerType], [TextHandler(string) for string in results] - ) - ) + return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results])) return TextHandlers( cast( @@ -257,9 +229,7 @@ class TextHandlers(List[TextHandler]): def __getitem__(self, pos: slice) -> "TextHandlers": # pragma: no cover pass - def __getitem__( - self, pos: SupportsIndex | slice - ) -> Union[TextHandler, "TextHandlers"]: + def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]: lst = super().__getitem__(pos) if isinstance(pos, slice): return TextHandlers(cast(List[_TextHandlerType], lst)) @@ -280,9 +250,7 @@ class TextHandlers(List[TextHandler]): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it """ - results = [ - n.re(regex, replace_entities, clean_match, case_sensitive) for n in self - ] + results = [n.re(regex, replace_entities, clean_match, case_sensitive) for n in self] return TextHandlers(flatten(results)) def re_first( @@ -330,34 +298,24 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): def __init__(self, mapping=None, **kwargs): mapping = ( - { - key: TextHandler(value) if isinstance(value, str) else value - for key, value in mapping.items() - } + {key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()} if mapping is not None else {} ) if kwargs: mapping.update( - { - key: TextHandler(value) if isinstance(value, str) else value - for key, value in kwargs.items() - } + {key: TextHandler(value) if isinstance(value, str) else value for key, value in kwargs.items()} ) # Fastest read-only mapping type self._data = MappingProxyType(mapping) - def get( - self, key: str, default: Optional[str] = None - ) -> Optional[_TextHandlerType]: + def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]: """Acts like the standard dictionary `.get()` method""" return self._data.get(key, default) - def search_values( - self, keyword: str, partial: bool = False - ) -> Generator["AttributesHandler", None, None]: + def search_values(self, keyword: str, partial: bool = False) -> Generator["AttributesHandler", None, None]: """Search current attributes by values and return a dictionary of each matching item :param keyword: The keyword to search for in the attribute values :param partial: If True, the function will search if keyword in each value instead of perfect match diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py index afad094..4087020 100644 --- a/scrapling/core/mixins.py +++ b/scrapling/core/mixins.py @@ -5,9 +5,7 @@ class SelectorsGeneration: Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591 """ - def __general_selection( - self, selection: str = "css", full_path: bool = False - ) -> str: + def __general_selection(self, selection: str = "css", full_path: bool = False) -> str: """Generate a selector for the current element. :return: A string of the generated selector. """ @@ -18,18 +16,10 @@ class SelectorsGeneration: if target.parent: if target.attrib.get("id"): # id is enough - part = ( - f"#{target.attrib['id']}" - if css - else f"[@id='{target.attrib['id']}']" - ) + part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']" selectorPath.append(part) if not full_path: - return ( - " > ".join(reversed(selectorPath)) - if css - else "//*" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath)) else: part = f"{target.tag}" # We won't use classes anymore because I some websites share exact classes between elements @@ -45,28 +35,16 @@ class SelectorsGeneration: break if counter[target.tag] > 1: - part += ( - f":nth-of-type({counter[target.tag]})" - if css - else f"[{counter[target.tag]}]" - ) + part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]" selectorPath.append(part) target = target.parent if target is None or target.tag == "html": - return ( - " > ".join(reversed(selectorPath)) - if css - else "//" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) else: break - return ( - " > ".join(reversed(selectorPath)) - if css - else "//" + "/".join(reversed(selectorPath)) - ) + return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) @property def generate_css_selector(self) -> str: diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 24e504c..8a38391 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -2,7 +2,6 @@ from re import sub as re_sub from sys import stderr from functools import wraps -from http import cookies as Cookie from collections import namedtuple from shlex import split as shlex_split from tempfile import mkstemp as make_temp_file @@ -23,25 +22,17 @@ from logging import ( from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ -from scrapling.core.custom_types import TextHandler -from scrapling.core.utils import log from scrapling.parser import Selector, Selectors +from scrapling.core.custom_types import TextHandler +from scrapling.engines.toolbelt.custom import Response +from scrapling.core.utils import log, _ParseHeaders, _CookieParser from scrapling.core._types import ( - List, Optional, Dict, - Tuple, Any, extraction_types, Generator, ) -from scrapling.fetchers import ( - Fetcher, - AsyncFetcher, - DynamicFetcher, - StealthyFetcher, - Response, -) _known_logging_levels = { @@ -71,54 +62,6 @@ Request = namedtuple( ) -def _CookieParser(cookie_string): - # Errors will be handled on call so the log can be specified - cookie_parser = Cookie.SimpleCookie() - cookie_parser.load(cookie_string) - for key, morsel in cookie_parser.items(): - yield key, morsel.value - - -def _ParseHeaders( - header_lines: List[str], parse_cookies: bool = True -) -> Tuple[Dict[str, str], Dict[str, str]]: - """Parses headers into separate header and cookie dictionaries.""" - header_dict = dict() - cookie_dict = dict() - - for header_line in header_lines: - if ":" not in header_line: - if header_line.endswith(";"): - header_key = header_line[:-1].strip() - header_value = "" - header_dict[header_key] = header_value - else: - raise ValueError( - f"Could not parse header without colon: '{header_line}'." - ) - else: - header_key, header_value = header_line.split(":", 1) - header_key = header_key.strip() - header_value = header_value.strip() - - if parse_cookies: - if header_key.lower() == "cookie": - try: - cookie_dict = { - key: value for key, value in _CookieParser(header_value) - } - except Exception as e: # pragma: no cover - raise ValueError( - f"Could not parse cookie string from header '{header_value}': {e}" - ) - else: - header_dict[header_key] = header_value - else: - header_dict[header_key] = header_value - - return header_dict, cookie_dict - - # Suppress exit on error to handle parsing errors gracefully class NoExitArgumentParser(ArgumentParser): # pragma: no cover def error(self, message): @@ -129,15 +72,16 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no cover if message: log.error(f"Scrapling shell exited with status {status}: {message}") self._print_message(message, stderr) - raise ValueError( - f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}" - ) + raise ValueError(f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}") class CurlParser: """Builds the argument parser for relevant curl flags from DevTools.""" def __init__(self): + from scrapling.fetchers import Fetcher as __Fetcher + + self.__fetcher = __Fetcher # We will use argparse parser to parse the curl command directly instead of regex # We will focus more on flags that will show up on curl commands copied from DevTools's network tab _parser = NoExitArgumentParser(add_help=False) # Disable default help @@ -152,15 +96,11 @@ class CurlParser: # Data arguments (prioritizing types common from DevTools) _parser.add_argument("-d", "--data", default=None) - _parser.add_argument( - "--data-raw", default=None - ) # Often used by browsers for JSON body + _parser.add_argument("--data-raw", default=None) # Often used by browsers for JSON body _parser.add_argument("--data-binary", default=None) # Keep urlencode for completeness, though less common from browser copy/paste _parser.add_argument("--data-urlencode", action="append", default=[]) - _parser.add_argument( - "-G", "--get", action="store_true" - ) # Use GET and put data in URL + _parser.add_argument("-G", "--get", action="store_true") # Use GET and put data in URL _parser.add_argument( "-b", @@ -175,9 +115,7 @@ class CurlParser: # Connection/Security _parser.add_argument("-k", "--insecure", action="store_true") - _parser.add_argument( - "--compressed", action="store_true" - ) # Very common from browsers + _parser.add_argument("--compressed", action="store_true") # Very common from browsers # Other flags often included but may not map directly to request args _parser.add_argument("-i", "--include", action="store_true") @@ -194,9 +132,7 @@ class CurlParser: clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ") try: - tokens = shlex_split( - clean_command - ) # Split the string using shell-like syntax + tokens = shlex_split(clean_command) # Split the string using shell-like syntax except ValueError as e: # pragma: no cover log.error(f"Could not split command line: {e}") return None @@ -213,9 +149,7 @@ class CurlParser: raise except Exception as e: # pragma: no cover - log.error( - f"An unexpected error occurred during curl arguments parsing: {e}" - ) + log.error(f"An unexpected error occurred during curl arguments parsing: {e}") return None # --- Determine Method --- @@ -247,9 +181,7 @@ class CurlParser: cookies[key] = value log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}") except Exception as e: # pragma: no cover - log.error( - f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}" - ) + log.error(f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}") # --- Process Data Payload --- params = dict() @@ -280,9 +212,7 @@ class CurlParser: try: data_payload = dict(parse_qsl(combined_data, keep_blank_values=True)) except Exception as e: - log.warning( - f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string." - ) + log.warning(f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string.") data_payload = combined_data # Check if raw data looks like JSON, prefer 'json' param if so @@ -303,9 +233,7 @@ class CurlParser: try: params.update(dict(parse_qsl(data_payload, keep_blank_values=True))) except ValueError: - log.warning( - f"Could not parse data '{data_payload}' into GET parameters for -G." - ) + log.warning(f"Could not parse data '{data_payload}' into GET parameters for -G.") if params: data_payload = None # Clear data as it's moved to params @@ -314,21 +242,13 @@ class CurlParser: # --- Process Proxy --- proxies: Optional[Dict[str, str]] = None if parsed_args.proxy: - proxy_url = ( - f"http://{parsed_args.proxy}" - if "://" not in parsed_args.proxy - else parsed_args.proxy - ) + proxy_url = f"http://{parsed_args.proxy}" if "://" not in parsed_args.proxy else parsed_args.proxy if parsed_args.proxy_user: user_pass = parsed_args.proxy_user parts = urlparse(proxy_url) netloc_parts = parts.netloc.split("@") - netloc = ( - f"{user_pass}@{netloc_parts[-1]}" - if len(netloc_parts) > 1 - else f"{user_pass}@{parts.netloc}" - ) + netloc = f"{user_pass}@{netloc_parts[-1]}" if len(netloc_parts) > 1 else f"{user_pass}@{parts.netloc}" proxy_url = urlunparse( ( parts.scheme, @@ -359,11 +279,7 @@ class CurlParser: def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]: if isinstance(curl_command, (Request, str)): - request = ( - self.parse(curl_command) - if isinstance(curl_command, str) - else curl_command - ) + request = self.parse(curl_command) if isinstance(curl_command, str) else curl_command # Ensure request parsing was successful before proceeding if request is None: # pragma: no cover @@ -381,14 +297,12 @@ class CurlParser: _ = request_args.pop("json", None) try: - return getattr(Fetcher, method)(**request_args) + return getattr(self.__fetcher, method)(**request_args) except Exception as e: # pragma: no cover log.error(f"Error calling Fetcher.{method}: {e}") return None else: # pragma: no cover - log.error( - f'Request method "{method}" isn\'t supported by Scrapling yet' - ) + log.error(f'Request method "{method}" isn\'t supported by Scrapling yet') return None else: # pragma: no cover @@ -403,7 +317,7 @@ def show_page_in_browser(page: Selector): # pragma: no cover try: fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html") - with open(fd, "w", encoding="utf-8") as f: + with open(fd, "w", encoding=page.encoding) as f: f.write(page.body) open_in_browser(f"file://{fname}") @@ -417,6 +331,19 @@ class CustomShell: """A custom IPython shell with minimal dependencies""" def __init__(self, code, log_level="debug"): + from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed + from scrapling.fetchers import ( + Fetcher as __Fetcher, + AsyncFetcher as __AsyncFetcher, + DynamicFetcher as __DynamicFetcher, + StealthyFetcher as __StealthyFetcher, + ) + + self.__InteractiveShellEmbed = __InteractiveShellEmbed + self.__Fetcher = __Fetcher + self.__AsyncFetcher = __AsyncFetcher + self.__DynamicFetcher = __DynamicFetcher + self.__StealthyFetcher = __StealthyFetcher self.code = code self.page = None self.pages = Selectors([]) @@ -440,7 +367,7 @@ class CustomShell: if self.log_level: getLogger("scrapling").setLevel(self.log_level) - settings = Fetcher.display_config() + settings = self.__Fetcher.display_config() settings.pop("storage", None) settings.pop("storage_args", None) log.info(f"Scrapling {__version__} shell started") @@ -506,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit. """Create a namespace with application-specific objects""" # Create wrapped versions of fetch functions - get = self.create_wrapper(Fetcher.get) - post = self.create_wrapper(Fetcher.post) - put = self.create_wrapper(Fetcher.put) - delete = self.create_wrapper(Fetcher.delete) - dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch) - stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch) + get = self.create_wrapper(self.__Fetcher.get) + post = self.create_wrapper(self.__Fetcher.post) + put = self.create_wrapper(self.__Fetcher.put) + delete = self.create_wrapper(self.__Fetcher.delete) + dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) + stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher) # Create the namespace dictionary @@ -520,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit. "post": post, "put": put, "delete": delete, - "Fetcher": Fetcher, - "AsyncFetcher": AsyncFetcher, + "Fetcher": self.__Fetcher, + "AsyncFetcher": self.__AsyncFetcher, "fetch": dynamic_fetch, - "DynamicFetcher": DynamicFetcher, + "DynamicFetcher": self.__DynamicFetcher, "stealthy_fetch": stealthy_fetch, - "StealthyFetcher": StealthyFetcher, + "StealthyFetcher": self.__StealthyFetcher, "Selector": Selector, "page": self.page, "response": self.page, @@ -542,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit. def start(self): # pragma: no cover """Start the interactive shell""" - from IPython.terminal.embed import InteractiveShellEmbed # Get our namespace with application objects namespace = self.get_namespace() - ipython_shell = InteractiveShellEmbed( + ipython_shell = self.__InteractiveShellEmbed( banner1=self.banner(), banner2="", enable_tip=False, @@ -621,20 +547,16 @@ class Convertor: yield "" @classmethod - def write_content_to_file( - cls, page: Selector, filename: str, css_selector: Optional[str] = None - ) -> None: + def write_content_to_file(cls, page: Selector, filename: str, css_selector: Optional[str] = None) -> None: """Write a Selector's content to a file""" if not page or not isinstance(page, Selector): # pragma: no cover raise TypeError("Input must be of type `Selector`") elif not filename or not isinstance(filename, str) or not filename.strip(): raise ValueError("Filename must be provided") elif not filename.endswith((".md", ".html", ".txt")): - raise ValueError( - "Unknown file type: filename must end with '.md', '.html', or '.txt'" - ) + raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'") else: - with open(filename, "w", encoding="utf-8") as f: + with open(filename, "w", encoding=page.encoding) as f: extension = filename.split(".")[-1] f.write( "".join( diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index 089a9ec..03c05a2 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -27,11 +27,7 @@ class StorageSystemMixin(ABC): # pragma: no cover try: extracted = tld(self.url) - return ( - extracted.top_domain_under_public_suffix - or extracted.domain - or default_value - ) + return extracted.top_domain_under_public_suffix or extracted.domain or default_value except AttributeError: return default_value @@ -90,9 +86,7 @@ class SQLiteStorageSystem(StorageSystemMixin): self.connection.execute("PRAGMA journal_mode=WAL") self.cursor = self.connection.cursor() self._setup_database() - log.debug( - f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")' - ) + log.debug(f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")') def _setup_database(self) -> None: self.cursor.execute(""" diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index e0a91bc..d98092e 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -10,10 +10,10 @@ So you don't have to learn a new selectors/api method like what bs4 done with so from functools import lru_cache -from cssselect import HTMLTranslator as OriginalHTMLTranslator -from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from cssselect.xpath import ExpressionError from cssselect.xpath import XPathExpr as OriginalXPathExpr +from cssselect import HTMLTranslator as OriginalHTMLTranslator +from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from scrapling.core._types import Any, Optional, Protocol, Self @@ -89,9 +89,7 @@ class TranslatorMixin: xpath = super().xpath_element(selector) # type: ignore[safe-super] return XPathExpr.from_xpath(xpath) - def xpath_pseudo_element( - self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement - ) -> OriginalXPathExpr: + def xpath_pseudo_element(self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement) -> OriginalXPathExpr: """ Dispatch method that transforms XPath to support the pseudo-element. """ @@ -99,31 +97,21 @@ class TranslatorMixin: method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element" method = getattr(self, method_name, None) if not method: # pragma: no cover - raise ExpressionError( - f"The functional pseudo-element ::{pseudo_element.name}() is unknown" - ) + raise ExpressionError(f"The functional pseudo-element ::{pseudo_element.name}() is unknown") xpath = method(xpath, pseudo_element) else: - method_name = ( - f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element" - ) + method_name = f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element" method = getattr(self, method_name, None) if not method: # pragma: no cover - raise ExpressionError( - f"The pseudo-element ::{pseudo_element} is unknown" - ) + raise ExpressionError(f"The pseudo-element ::{pseudo_element} is unknown") xpath = method(xpath) return xpath @staticmethod - def xpath_attr_functional_pseudo_element( - xpath: OriginalXPathExpr, function: FunctionalPseudoElement - ) -> XPathExpr: + def xpath_attr_functional_pseudo_element(xpath: OriginalXPathExpr, function: FunctionalPseudoElement) -> XPathExpr: """Support selecting attribute values using ::attr() pseudo-element""" if function.argument_types() not in (["STRING"], ["IDENT"]): # pragma: no cover - raise ExpressionError( - f"Expected a single string or ident for ::attr(), got {function.arguments!r}" - ) + raise ExpressionError(f"Expected a single string or ident for ::attr(), got {function.arguments!r}") return XPathExpr.from_xpath(xpath, attribute=function.arguments[0].value) @staticmethod diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.py new file mode 100644 index 0000000..dc95705 --- /dev/null +++ b/scrapling/core/utils/__init__.py @@ -0,0 +1,10 @@ +from ._utils import ( + log, + __CONSECUTIVE_SPACES_REGEX__, + flatten, + _is_iterable, + _StorageTools, + clean_spaces, + html_forbidden, +) +from ._shell import _CookieParser, _ParseHeaders diff --git a/scrapling/core/utils/_shell.py b/scrapling/core/utils/_shell.py new file mode 100644 index 0000000..05420ae --- /dev/null +++ b/scrapling/core/utils/_shell.py @@ -0,0 +1,48 @@ +from http import cookies as Cookie + + +from scrapling.core._types import ( + List, + Dict, + Tuple, +) + + +def _CookieParser(cookie_string): + # Errors will be handled on call so the log can be specified + cookie_parser = Cookie.SimpleCookie() + cookie_parser.load(cookie_string) + for key, morsel in cookie_parser.items(): + yield key, morsel.value + + +def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]: + """Parses headers into separate header and cookie dictionaries.""" + header_dict = dict() + cookie_dict = dict() + + for header_line in header_lines: + if ":" not in header_line: + if header_line.endswith(";"): + header_key = header_line[:-1].strip() + header_value = "" + header_dict[header_key] = header_value + else: + raise ValueError(f"Could not parse header without colon: '{header_line}'.") + else: + header_key, header_value = header_line.split(":", 1) + header_key = header_key.strip() + header_value = header_value.strip() + + if parse_cookies: + if header_key.lower() == "cookie": + try: + cookie_dict = {key: value for key, value in _CookieParser(header_value)} + except Exception as e: # pragma: no cover + raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}") + else: + header_dict[header_key] = header_value + else: + header_dict[header_key] = header_value + + return header_dict, cookie_dict diff --git a/scrapling/core/utils.py b/scrapling/core/utils/_utils.py similarity index 78% rename from scrapling/core/utils.py rename to scrapling/core/utils/_utils.py index 5607f8d..2f57cfa 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils/_utils.py @@ -24,9 +24,7 @@ def setup_logger(): logger = logging.getLogger("scrapling") logger.setLevel(logging.INFO) - formatter = logging.Formatter( - fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S" - ) + formatter = logging.Formatter(fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) @@ -61,11 +59,7 @@ class _StorageTools: def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict: if not element.attrib: return {} - return { - k: v.strip() - for k, v in element.attrib.items() - if v and v.strip() and k not in forbidden - } + return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden} @classmethod def element_to_dict(cls, element: html.HtmlElement) -> Dict: @@ -85,17 +79,11 @@ class _StorageTools: } ) - siblings = [ - child.tag for child in parent.iterchildren() if child != element - ] + siblings = [child.tag for child in parent.iterchildren() if child != element] if siblings: result.update({"siblings": tuple(siblings)}) - children = [ - child.tag - for child in element.iterchildren() - if not isinstance(child, html_forbidden) - ] + children = [child.tag for child in element.iterchildren() if not isinstance(child, html_forbidden)] if children: result.update({"children": tuple(children)}) @@ -104,11 +92,7 @@ class _StorageTools: @classmethod def _get_element_path(cls, element: html.HtmlElement): parent = element.getparent() - return tuple( - (element.tag,) - if parent is None - else (cls._get_element_path(parent) + (element.tag,)) - ) + return tuple((element.tag,) if parent is None else (cls._get_element_path(parent) + (element.tag,))) @lru_cache(128, typed=True) diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index 477bb21..e69de29 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,16 +0,0 @@ -from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS -from .static import FetcherSession, FetcherClient, AsyncFetcherClient -from ._browsers import ( - DynamicSession, - AsyncDynamicSession, - StealthySession, - AsyncStealthySession, -) - -__all__ = [ - "FetcherSession", - "DynamicSession", - "AsyncDynamicSession", - "StealthySession", - "AsyncStealthySession", -] diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py new file mode 100644 index 0000000..00c5567 --- /dev/null +++ b/scrapling/engines/_browsers/_base.py @@ -0,0 +1,297 @@ +from time import time, sleep +from asyncio import sleep as asyncio_sleep, Lock + +from camoufox import DefaultAddons +from playwright.sync_api import BrowserContext, Playwright +from playwright.async_api import ( + BrowserContext as AsyncBrowserContext, + Playwright as AsyncPlaywright, +) +from camoufox.utils import ( + launch_options as generate_launch_options, + installed_verstr as camoufox_version, +) + +from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route +from scrapling.core._types import ( + Any, + Dict, + Optional, +) +from ._page import PageInfo, PagePool +from ._config_tools import _compiled_stealth_scripts +from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.fingerprints import get_os_name +from ._validators import validate, PlaywrightConfig, CamoufoxConfig + +__ff_version_str__ = camoufox_version().split(".", 1)[0] + + +class SyncSession: + def __init__(self, max_pages: int = 1): + self.max_pages = max_pages + self.page_pool = PagePool(max_pages) + self.__max_wait_for_page = 60 + self.playwright: Optional[Playwright] = None + self.context: Optional[BrowserContext] = None + self._closed = False + + def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover + """Get a new page to use""" + + # Close all finished pages to ensure clean state + self.page_pool.close_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + sleep(0.05) + self.page_pool.close_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = self.context.new_page() + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if extra_headers: + page.set_extra_http_headers(extra_headers) + + if disable_resources: + page.route("**/*", intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + @staticmethod + def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any: + """Get value with request-level priority over session-level""" + return request_value if request_value is not sentinel_value else session_value + + def get_pool_stats(self) -> Dict[str, int]: + """Get statistics about the current page pool""" + return { + "total_pages": self.page_pool.pages_count, + "busy_pages": self.page_pool.busy_count, + "max_pages": self.max_pages, + } + + +class AsyncSession(SyncSession): + def __init__(self, max_pages: int = 1): + super().__init__(max_pages) + self.playwright: Optional[AsyncPlaywright] = None + self.context: Optional[AsyncBrowserContext] = None + self._lock = Lock() + + async def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover + """Get a new page to use""" + async with self._lock: + # Close all finished pages to ensure clean state + await self.page_pool.aclose_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + await asyncio_sleep(0.05) + await self.page_pool.aclose_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = await self.context.new_page() + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if extra_headers: + await page.set_extra_http_headers(extra_headers) + + if disable_resources: + await page.route("**/*", async_intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + await page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + +class DynamicSessionMixin: + def __validate__(self, **params): + config = validate(params, model=PlaywrightConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.hide_canvas = config.hide_canvas + self.disable_webgl = config.disable_webgl + self.real_chrome = config.real_chrome + self.stealth = config.stealth + self.google_search = config.google_search + self.wait = config.wait + self.proxy = config.proxy + self.locale = config.locale + self.extra_headers = config.extra_headers + self.useragent = config.useragent + self.timeout = config.timeout + self.cookies = config.cookies + self.disable_resources = config.disable_resources + self.cdp_url = config.cdp_url + self.network_idle = config.network_idle + self.load_dom = config.load_dom + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.wait_selector_state = config.wait_selector_state + self.selector_config = config.selector_config + self.page_action = config.page_action + self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + if not self.cdp_url: + # `launch_options` is used with persistent context + self.launch_options = dict( + _launch_kwargs( + self.headless, + self.proxy, + self.locale, + tuple(self.extra_headers.items()) if self.extra_headers else tuple(), + self.useragent, + self.real_chrome, + self.stealth, + self.hide_canvas, + self.disable_webgl, + ) + ) + self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) + self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + self.context_options = dict() + else: + # while `context_options` is left to be used when cdp mode is enabled + self.launch_options = dict() + self.context_options = dict( + _context_kwargs( + self.proxy, + self.locale, + tuple(self.extra_headers.items()) if self.extra_headers else tuple(), + self.useragent, + self.stealth, + ) + ) + self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) + self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + + +class StealthySessionMixin: + def __validate__(self, **params): + config = validate(params, model=CamoufoxConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.block_images = config.block_images + self.disable_resources = config.disable_resources + self.block_webrtc = config.block_webrtc + self.allow_webgl = config.allow_webgl + self.network_idle = config.network_idle + self.load_dom = config.load_dom + self.humanize = config.humanize + self.solve_cloudflare = config.solve_cloudflare + self.wait = config.wait + self.timeout = config.timeout + self.page_action = config.page_action + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.addons = config.addons + self.wait_selector_state = config.wait_selector_state + self.cookies = config.cookies + self.google_search = config.google_search + self.extra_headers = config.extra_headers + self.proxy = config.proxy + self.os_randomize = config.os_randomize + self.disable_ads = config.disable_ads + self.geoip = config.geoip + self.selector_config = config.selector_config + self.additional_args = config.additional_args + self.page_action = config.page_action + self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + """Initiate browser options.""" + self.launch_options = generate_launch_options( + **{ + "geoip": self.geoip, + "proxy": dict(self.proxy) if self.proxy else self.proxy, + "addons": self.addons, + "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], + "headless": self.headless, + "humanize": True if self.solve_cloudflare else self.humanize, + "i_know_what_im_doing": True, # To turn warnings off with the user configurations + "allow_webgl": self.allow_webgl, + "block_webrtc": self.block_webrtc, + "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. + "os": None if self.os_randomize else get_os_name(), + "user_data_dir": "", + "ff_version": __ff_version_str__, + "firefox_user_prefs": { + # This is what enabling `enable_cache` does internally, so we do it from here instead + "browser.sessionhistory.max_entries": 10, + "browser.sessionhistory.max_total_viewers": -1, + "browser.cache.memory.enable": True, + "browser.cache.disk_cache_ssl": True, + "browser.cache.disk.smart_size.enabled": True, + }, + **self.additional_args, + } + ) + + @staticmethod + def _detect_cloudflare(page_content: str) -> str | None: + """ + Detect the type of Cloudflare challenge present in the provided page content. + + This function analyzes the given page content to identify whether a specific + type of Cloudflare challenge is present. It checks for three predefined + challenge types: non-interactive, managed, and interactive. If a challenge + type is detected, it returns the corresponding type as a string. If no + challenge type is detected, it returns None. + + Args: + page_content (str): The content of the page to analyze for Cloudflare + challenge types. + + Returns: + str: A string representing the detected Cloudflare challenge type, if + found. Returns None if no challenge matches. + """ + challenge_types = ( + "non-interactive", + "managed", + "interactive", + ) + for ctype in challenge_types: + if f"cType: '{ctype}'" in page_content: + return ctype + + return None diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index d6150e4..6e0bc1e 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -1,14 +1,8 @@ -from time import time, sleep from re import compile as re_compile -from asyncio import sleep as asyncio_sleep, Lock -from camoufox import DefaultAddons -from camoufox.utils import launch_options as generate_launch_options from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, - Playwright, Locator, Page, ) @@ -21,9 +15,9 @@ from playwright.async_api import ( Page as async_Page, ) -from scrapling.core.utils import log -from ._page import PageInfo, PagePool from ._validators import validate, CamoufoxConfig +from ._base import SyncSession, AsyncSession, StealthySessionMixin +from scrapling.core.utils import log from scrapling.core._types import ( Dict, List, @@ -31,19 +25,17 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - async_intercept_route, - generate_convincing_referer, - get_os_name, - intercept_route, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +_UNSET = object() -class StealthySession: +class StealthySession(StealthySessionMixin, SyncSession): """A Stealthy session manager with page pooling.""" __slots__ = ( @@ -54,6 +46,7 @@ class StealthySession: "block_webrtc", "allow_webgl", "network_idle", + "load_dom", "humanize", "solve_cloudflare", "wait", @@ -83,13 +76,14 @@ class StealthySession: def __init__( self, - max_pages: int = 1, + __max_pages: int = 1, headless: bool = True, # noqa: F821 block_images: bool = False, disable_resources: bool = False, block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -124,11 +118,12 @@ class StealthySession: :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -137,108 +132,51 @@ class StealthySession: :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - params = { - "max_pages": max_pages, - "headless": headless, - "block_images": block_images, - "disable_resources": disable_resources, - "block_webrtc": block_webrtc, - "allow_webgl": allow_webgl, - "network_idle": network_idle, - "humanize": humanize, - "solve_cloudflare": solve_cloudflare, - "wait": wait, - "timeout": timeout, - "page_action": page_action, - "wait_selector": wait_selector, - "init_script": init_script, - "addons": addons, - "wait_selector_state": wait_selector_state, - "cookies": cookies, - "google_search": google_search, - "extra_headers": extra_headers, - "proxy": proxy, - "os_randomize": os_randomize, - "disable_ads": disable_ads, - "geoip": geoip, - "selector_config": selector_config, - "additional_args": additional_args, - } - config = validate(params, CamoufoxConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.block_images = config.block_images - self.disable_resources = config.disable_resources - self.block_webrtc = config.block_webrtc - self.allow_webgl = config.allow_webgl - self.network_idle = config.network_idle - self.humanize = config.humanize - self.solve_cloudflare = config.solve_cloudflare - self.wait = config.wait - self.timeout = config.timeout - self.page_action = config.page_action - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.addons = config.addons - self.wait_selector_state = config.wait_selector_state - self.cookies = config.cookies - self.google_search = config.google_search - self.extra_headers = config.extra_headers - self.proxy = config.proxy - self.os_randomize = config.os_randomize - self.disable_ads = config.disable_ads - self.geoip = config.geoip - self.selector_config = config.selector_config - self.additional_args = config.additional_args - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() - ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - """Initiate browser options.""" - self.launch_options = generate_launch_options( - **{ - "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy else self.proxy, - "enable_cache": True, - "addons": self.addons, - "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], - "headless": self.headless, - "humanize": True if self.solve_cloudflare else self.humanize, - "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self.allow_webgl, - "block_webrtc": self.block_webrtc, - "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self.os_randomize else get_os_name(), - "user_data_dir": "", - **self.additional_args, - } + self.__validate__( + wait=wait, + proxy=proxy, + geoip=geoip, + addons=addons, + timeout=timeout, + cookies=cookies, + headless=headless, + humanize=humanize, + load_dom=load_dom, + max_pages=__max_pages, + disable_ads=disable_ads, + allow_webgl=allow_webgl, + page_action=page_action, + init_script=init_script, + network_idle=network_idle, + block_images=block_images, + block_webrtc=block_webrtc, + os_randomize=os_randomize, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + additional_args=additional_args, + selector_config=selector_config, + solve_cloudflare=solve_cloudflare, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() - self.context = ( - self.playwright.firefox.launch_persistent_context( # pragma: no cover - **self.launch_options - ) + self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover + **self.launch_options ) + + # Get the default page and close it + default_page = self.context.pages[0] + default_page.close() + if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -267,68 +205,6 @@ class StealthySession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - @staticmethod - def _detect_cloudflare(page_content): - """ - Detect the type of Cloudflare challenge present in the provided page content. - - This function analyzes the given page content to identify whether a specific - type of Cloudflare challenge is present. It checks for three predefined - challenge types: non-interactive, managed, and interactive. If a challenge - type is detected, it returns the corresponding type as a string. If no - challenge type is detected, it returns None. - - Args: - page_content (str): The content of the page to analyze for Cloudflare - challenge types. - - Returns: - str: A string representing the detected Cloudflare challenge type, if - found. Returns None if no challenge matches. - """ - challenge_types = ( - "non-interactive", - "managed", - "interactive", - ) - for ctype in challenge_types: - if f"cType: '{ctype}'" in page_content: - return ctype - - return None - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -375,20 +251,66 @@ class StealthySession: log.info("Cloudflare captcha is solved") return - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + load_dom: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( - generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -399,54 +321,57 @@ class StealthySession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if params.solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if params.page_action: try: - page_info.page = self.page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -454,17 +379,8 @@ class StealthySession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncStealthySession(StealthySession): +class AsyncStealthySession(StealthySessionMixin, AsyncSession): """A Stealthy session manager with page pooling.""" def __init__( @@ -476,6 +392,7 @@ class AsyncStealthySession(StealthySession): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -510,11 +427,12 @@ class AsyncStealthySession(StealthySession): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -527,47 +445,47 @@ class AsyncStealthySession(StealthySession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - super().__init__( - max_pages, - headless, - block_images, - disable_resources, - block_webrtc, - allow_webgl, - network_idle, - humanize, - solve_cloudflare, - wait, - timeout, - page_action, - wait_selector, - init_script, - addons, - wait_selector_state, - cookies, - google_search, - extra_headers, - proxy, - os_randomize, - disable_ads, - geoip, - selector_config, - additional_args, + self.__validate__( + wait=wait, + proxy=proxy, + geoip=geoip, + addons=addons, + timeout=timeout, + cookies=cookies, + headless=headless, + load_dom=load_dom, + humanize=humanize, + max_pages=max_pages, + disable_ads=disable_ads, + allow_webgl=allow_webgl, + page_action=page_action, + init_script=init_script, + network_idle=network_idle, + block_images=block_images, + block_webrtc=block_webrtc, + os_randomize=os_randomize, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + additional_args=additional_args, + selector_config=selector_config, + solve_cloudflare=solve_cloudflare, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = ( - await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( + **self.launch_options ) + + # Get the default page and close it + default_page = self.context.pages[0] + await default_page.close() + if self.init_script: # pragma: no cover await self.context.add_init_script(path=self.init_script) @@ -596,39 +514,6 @@ class AsyncStealthySession(StealthySession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version @@ -664,9 +549,7 @@ class AsyncStealthySession(StealthySession): await page.wait_for_timeout(500) # Calculate the Captcha coordinates for any viewport - outer_box = await page.locator( - ".main-content p+div>div>div" - ).bounding_box() + outer_box = await page.locator(".main-content p+div>div>div").bounding_box() captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 # Move the mouse to the center of the window, then press and hold the left mouse button @@ -677,20 +560,65 @@ class AsyncStealthySession(StealthySession): log.info("Cloudflare captcha is solved") return - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + load_dom: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( - generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -701,56 +629,59 @@ class AsyncStealthySession(StealthySession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if params.solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if params.page_action: try: - page_info.page = await self.page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 96b7f41..4b405e6 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -6,7 +6,8 @@ from scrapling.engines.constants import ( HARMFUL_DEFAULT_ARGS, DEFAULT_FLAGS, ) -from scrapling.engines.toolbelt import js_bypass_path, generate_headers +from scrapling.engines.toolbelt.navigation import js_bypass_path +from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 2747ed9..07efee2 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -1,10 +1,6 @@ -from time import time, sleep -from asyncio import sleep as asyncio_sleep, Lock - from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, Playwright, Locator, ) @@ -21,9 +17,8 @@ from rebrowser_playwright.async_api import ( ) from scrapling.core.utils import log -from ._page import PageInfo, PagePool +from ._base import SyncSession, AsyncSession, DynamicSessionMixin from ._validators import validate, PlaywrightConfig -from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs from scrapling.core._types import ( Dict, List, @@ -31,16 +26,16 @@ from scrapling.core._types import ( Callable, SelectorWaitStates, ) -from scrapling.engines.toolbelt import ( +from scrapling.engines.toolbelt.convertor import ( Response, ResponseFactory, - generate_convincing_referer, - intercept_route, - async_intercept_route, ) +from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer + +_UNSET = object() -class DynamicSession: +class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" __slots__ = ( @@ -59,6 +54,7 @@ class DynamicSession: "cookies", "disable_resources", "network_idle", + "load_dom", "wait_selector", "init_script", "wait_selector_state", @@ -98,6 +94,7 @@ class DynamicSession: init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -112,7 +109,7 @@ class DynamicSession: :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -121,114 +118,39 @@ class DynamicSession: :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - - params = { - "max_pages": __max_pages, - "headless": headless, - "google_search": google_search, - "hide_canvas": hide_canvas, - "disable_webgl": disable_webgl, - "real_chrome": real_chrome, - "stealth": stealth, - "wait": wait, - "page_action": page_action, - "proxy": proxy, - "locale": locale, - "extra_headers": extra_headers, - "useragent": useragent, - "timeout": timeout, - "selector_config": selector_config, - "disable_resources": disable_resources, - "wait_selector": wait_selector, - "init_script": init_script, - "cookies": cookies, - "network_idle": network_idle, - "wait_selector_state": wait_selector_state, - "cdp_url": cdp_url, - } - config = validate(params, PlaywrightConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.hide_canvas = config.hide_canvas - self.disable_webgl = config.disable_webgl - self.real_chrome = config.real_chrome - self.stealth = config.stealth - self.google_search = config.google_search - self.wait = config.wait - self.proxy = config.proxy - self.locale = config.locale - self.extra_headers = config.extra_headers - self.useragent = config.useragent - self.timeout = config.timeout - self.cookies = config.cookies - self.disable_resources = config.disable_resources - self.cdp_url = config.cdp_url - self.network_idle = config.network_idle - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.wait_selector_state = config.wait_selector_state - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() + self.__validate__( + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + stealth=stealth, + cdp_url=cdp_url, + cookies=cookies, + load_dom=load_dom, + headless=headless, + useragent=useragent, + max_pages=__max_pages, + real_chrome=real_chrome, + page_action=page_action, + hide_canvas=hide_canvas, + init_script=init_script, + network_idle=network_idle, + google_search=google_search, + extra_headers=extra_headers, + wait_selector=wait_selector, + disable_webgl=disable_webgl, + selector_config=selector_config, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - if not self.cdp_url: - # `launch_options` is used with persistent context - self.launch_options = dict( - _launch_kwargs( - self.headless, - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.real_chrome, - self.stealth, - self.hide_canvas, - self.disable_webgl, - ) - ) - self.launch_options["extra_http_headers"] = dict( - self.launch_options["extra_http_headers"] - ) - self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.context_options = dict() - else: - # while `context_options` is left to be used when cdp mode is enabled - self.launch_options = dict() - self.context_options = dict( - _context_kwargs( - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.stealth, - ) - ) - self.context_options["extra_http_headers"] = dict( - self.context_options["extra_http_headers"] - ) - self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" @@ -237,16 +159,18 @@ class DynamicSession: # Because rebrowser_playwright doesn't play well with real browsers sync_context = sync_playwright - self.playwright = sync_context().start() + self.playwright: Playwright = sync_context().start() if self.cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ).new_context(**self.context_options) - else: - self.context = self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( + **self.context_options ) + else: + self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options) + + # Get the default page and close it + default_page = self.context.pages[0] + default_page.close() if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -276,56 +200,63 @@ class DynamicSession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + load_dom: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( - generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -336,48 +267,50 @@ class DynamicSession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if params.page_action: try: - page_info.page = self.page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(params.wait) # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -385,17 +318,8 @@ class DynamicSession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncDynamicSession(DynamicSession): +class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" def __init__( @@ -420,6 +344,7 @@ class AsyncDynamicSession(DynamicSession): init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -432,9 +357,10 @@ class AsyncDynamicSession(DynamicSession): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -451,36 +377,32 @@ class AsyncDynamicSession(DynamicSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - super().__init__( - max_pages, - headless, - google_search, - hide_canvas, - disable_webgl, - real_chrome, - stealth, - wait, - page_action, - proxy, - locale, - extra_headers, - useragent, - cdp_url, - timeout, - disable_resources, - wait_selector, - init_script, - cookies, - network_idle, - wait_selector_state, - selector_config, + self.__validate__( + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + stealth=stealth, + cdp_url=cdp_url, + cookies=cookies, + load_dom=load_dom, + headless=headless, + useragent=useragent, + max_pages=max_pages, + real_chrome=real_chrome, + page_action=page_action, + hide_canvas=hide_canvas, + init_script=init_script, + network_idle=network_idle, + google_search=google_search, + extra_headers=extra_headers, + wait_selector=wait_selector, + disable_webgl=disable_webgl, + selector_config=selector_config, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, ) - - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -492,19 +414,17 @@ class AsyncDynamicSession(DynamicSession): self.playwright: AsyncPlaywright = await async_context().start() if self.cdp_url: - browser = await self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ) - self.context: AsyncBrowserContext = await browser.new_context( - **self.context_options - ) + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: - self.context: AsyncBrowserContext = ( - await self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + user_data_dir="", **self.launch_options ) + # Get the default page and close it + default_page = self.context.pages[0] + await default_page.close() + if self.init_script: # pragma: no cover await self.context.add_init_script(path=self.init_script) @@ -533,57 +453,63 @@ class AsyncDynamicSession(DynamicSession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - await page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 # seconds - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + load_dom: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( - generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -594,48 +520,50 @@ class AsyncDynamicSession(DynamicSession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if params.page_action: try: - page_info.page = await self.page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index ec418d0..ffa4b22 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,7 +6,7 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal["ready", "busy", "error"] # States that a page can be in +PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in @dataclass @@ -23,9 +23,9 @@ class PageInfo: self.state = "busy" self.url = url - def mark_ready(self): - """Mark the page as ready for new requests""" - self.state = "ready" + def mark_finished(self): + """Mark the page as finished for new requests""" + self.state = "finished" self.url = "" def mark_error(self): @@ -62,24 +62,16 @@ class PagePool: self.pages.append(page_info) return page_info - def get_ready_page(self) -> Optional[PageInfo]: - """Get a page that's ready for use""" - with self._lock: - for page_info in self.pages: - if page_info.state == "ready": - return page_info - return None - @property def pages_count(self) -> int: """Get the total number of pages""" return len(self.pages) @property - def ready_count(self) -> int: - """Get the number of ready pages""" + def finished_count(self) -> int: + """Get the number of finished pages""" with self._lock: - return sum(1 for p in self.pages if p.state == "ready") + return sum(1 for p in self.pages if p.state == "finished") @property def busy_count(self) -> int: @@ -91,3 +83,33 @@ class PagePool: """Remove pages in error state""" with self._lock: self.pages = [p for p in self.pages if p.state != "error"] + + def close_all_finished_pages(self): + """Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) + + async def aclose_all_finished_pages(self): + """Async version: Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + await page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 6363df7..ca64942 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -9,7 +9,7 @@ from scrapling.core._types import ( List, SelectorWaitStates, ) -from scrapling.engines.toolbelt import construct_proxy_dict +from scrapling.engines.toolbelt.navigation import construct_proxy_dict class PlaywrightConfig(Struct, kw_only=True, frozen=False): @@ -25,9 +25,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): stealth: bool = False wait: int | float = 0 page_action: Optional[Callable] = None - proxy: Optional[str | Dict[str, str]] = ( - None # The default value for proxy in Playwright's source is `None` - ) + proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` locale: str = "en-US" extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None @@ -37,6 +35,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): wait_selector: Optional[str] = None cookies: Optional[List[Dict]] = None network_idle: bool = False + load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" selector_config: Optional[Dict] = None @@ -46,10 +45,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): raise ValueError("max_pages must be between 1 and 50") if self.timeout < 0: raise ValueError("timeout must be >= 0") - if self.page_action is not None and not callable(self.page_action): - raise TypeError( - f"page_action must be callable, got {type(self.page_action).__name__}" - ) + if self.page_action and not callable(self.page_action): + raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) if self.cdp_url: @@ -96,6 +93,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): block_webrtc: bool = False allow_webgl: bool = True network_idle: bool = False + load_dom: bool = True humanize: bool | float = True solve_cloudflare: bool = False wait: int | float = 0 @@ -108,9 +106,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): cookies: Optional[List[Dict]] = None google_search: bool = True extra_headers: Optional[Dict[str, str]] = None - proxy: Optional[str | Dict[str, str]] = ( - None # The default value for proxy in Playwright's source is `None` - ) + proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None` os_randomize: bool = False disable_ads: bool = False geoip: bool = False @@ -123,10 +119,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): raise ValueError("max_pages must be between 1 and 50") if self.timeout < 0: raise ValueError("timeout must be >= 0") - if self.page_action is not None and not callable(self.page_action): - raise TypeError( - f"page_action must be callable, got {type(self.page_action).__name__}" - ) + if self.page_action and not callable(self.page_action): + raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py index c7bdb1b..03a678e 100644 --- a/scrapling/engines/constants.py +++ b/scrapling/engines/constants.py @@ -16,9 +16,9 @@ HARMFUL_DEFAULT_ARGS = ( # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884 "--enable-automation", "--disable-popup-blocking", - # '--disable-component-update', - # '--disable-default-apps', - # '--disable-extensions', + "--disable-component-update", + "--disable-default-apps", + "--disable-extensions", ) DEFAULT_FLAGS = ( @@ -50,7 +50,6 @@ DEFAULT_STEALTH_FLAGS = ( "--accept-lang=en-US", "--use-mock-keychain", "--disable-translate", - "--disable-extensions", "--disable-voice-input", "--window-position=0,0", "--disable-wake-on-wifi", @@ -59,7 +58,6 @@ DEFAULT_STEALTH_FLAGS = ( "--enable-web-bluetooth", "--disable-hang-monitor", "--disable-cloud-import", - "--disable-default-apps", "--disable-print-preview", "--disable-dev-shm-usage", # '--disable-popup-blocking', @@ -72,7 +70,6 @@ DEFAULT_STEALTH_FLAGS = ( "--force-color-profile=srgb", "--font-render-hinting=none", "--aggressive-cache-discard", - "--disable-component-update", "--disable-cookie-encryption", "--disable-domain-reliability", "--disable-threaded-animation", diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 59f68fa..3f6cb79 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -24,13 +24,9 @@ from scrapling.core._types import ( Any, ) -from .toolbelt import ( - Response, - generate_convincing_referer, - generate_headers, - ResponseFactory, - __default_useragent__, -) +from .toolbelt.custom import Response +from .toolbelt.convertor import ResponseFactory +from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ _UNSET = object() @@ -108,13 +104,9 @@ class FetcherSession: headers = self.get_with_precedence(kwargs, "headers", self.default_headers) stealth = self.get_with_precedence(kwargs, "stealth", self.stealth) - impersonate = self.get_with_precedence( - kwargs, "impersonate", self.default_impersonate - ) + impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate) - if self.get_with_precedence( - kwargs, "http3", self.default_http3 - ): # pragma: no cover + if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover request_args["http_version"] = CurlHttpVersion.V3ONLY if impersonate: log.warning( @@ -126,25 +118,13 @@ class FetcherSession: "url": url, # Curl automatically generates the suitable browser headers when you use `impersonate` "headers": self._headers_job(url, headers, stealth, bool(impersonate)), - "proxies": self.get_with_precedence( - kwargs, "proxies", self.default_proxies - ), + "proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies), "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy), - "proxy_auth": self.get_with_precedence( - kwargs, "proxy_auth", self.default_proxy_auth - ), - "timeout": self.get_with_precedence( - kwargs, "timeout", self.default_timeout - ), - "allow_redirects": self.get_with_precedence( - kwargs, "allow_redirects", self.default_follow_redirects - ), - "max_redirects": self.get_with_precedence( - kwargs, "max_redirects", self.default_max_redirects - ), - "verify": self.get_with_precedence( - kwargs, "verify", self.default_verify - ), + "proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth), + "timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout), + "allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects), + "max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects), + "verify": self.get_with_precedence(kwargs, "verify", self.default_verify), "cert": self.get_with_precedence(kwargs, "cert", self.default_cert), "impersonate": impersonate, **{ @@ -192,18 +172,12 @@ class FetcherSession: extra_headers = generate_headers(browser_mode=False) # Don't overwrite user-supplied headers - extra_headers = { - key: value - for key, value in extra_headers.items() - if key.lower() not in headers_keys - } + extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} headers.update(extra_headers) elif "user-agent" not in headers_keys and not impersonate_enabled: headers["User-Agent"] = __default_useragent__ - log.debug( - f"Can't find useragent in headers so '{headers['User-Agent']}' was used." - ) + log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") return headers @@ -215,9 +189,7 @@ class FetcherSession: "Create a new FetcherSession instance for a new independent session, " "or use the current instance sequentially after the previous context has exited." ) - if ( - self._async_curl_session - ): # Prevent mixing if async is active from this instance + if self._async_curl_session: # Prevent mixing if async is active from this instance raise RuntimeError( "This FetcherSession instance has an active asynchronous session. " "Cannot enter a synchronous context simultaneously with the same manager instance." @@ -275,9 +247,7 @@ class FetcherSession: :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ session = self._curl_session - if session is True and not any( - (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) - ): + if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): # For usage inside FetcherClient # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. session = CurlSession() @@ -290,9 +260,7 @@ class FetcherSession: return ResponseFactory.from_http_request(response, selector_config) except CurlError as e: # pragma: no cover if attempt < max_retries - 1: - log.error( - f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." - ) + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") time_sleep(retry_delay) else: log.error(f"Failed after {max_retries} attempts: {e}") @@ -320,9 +288,7 @@ class FetcherSession: :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ session = self._async_curl_session - if session is True and not any( - (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) - ): + if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. # 2. `curl_cffi` doesn't support making async requests without sessions @@ -337,9 +303,7 @@ class FetcherSession: return ResponseFactory.from_http_request(response, selector_config) except CurlError as e: # pragma: no cover if attempt < max_retries - 1: - log.error( - f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." - ) + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") await asyncio_sleep(retry_delay) else: log.error(f"Failed after {max_retries} attempts: {e}") @@ -372,19 +336,13 @@ class FetcherSession: selector_config = kwargs.pop("selector_config", {}) or self.selector_config max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) - retry_delay = self.get_with_precedence( - kwargs, "retry_delay", self.default_retry_delay - ) + retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay) request_args = self._merge_request_args(stealth=stealth, **kwargs) if self._curl_session: - return self.__make_request( - method, request_args, max_retries, retry_delay, selector_config - ) + return self.__make_request(method, request_args, max_retries, retry_delay, selector_config) elif self._async_curl_session: # The returned value is a Coroutine - return self.__make_async_request( - method, request_args, max_retries, retry_delay, selector_config - ) + return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config) raise RuntimeError("No active session available.") @@ -455,9 +413,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "GET", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args) def post( self, @@ -532,9 +488,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "POST", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args) def put( self, @@ -609,9 +563,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "PUT", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args) def delete( self, @@ -688,9 +640,7 @@ class FetcherSession: "http3": http3, **kwargs, } - return self.__prepare_and_dispatch( - "DELETE", stealth=stealthy_headers, **request_args - ) + return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args) class FetcherClient(FetcherSession): diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index d58fd57..8b13789 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -1,20 +1 @@ -from .custom import ( - BaseFetcher, - Response, - StatusText, - get_variable_name, -) -from .fingerprints import ( - generate_convincing_referer, - generate_headers, - get_os_name, - __default_useragent__, -) -from .navigation import ( - async_intercept_route, - construct_cdp_url, - construct_proxy_dict, - intercept_route, - js_bypass_path, -) -from .convertor import ResponseFactory + diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 12bfd55..05dce87 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -1,10 +1,15 @@ +from functools import lru_cache +from re import compile as re_compile + from curl_cffi.requests import Response as CurlResponse from playwright.sync_api import Page as SyncPage, Response as SyncResponse from playwright.async_api import Page as AsyncPage, Response as AsyncResponse from scrapling.core.utils import log -from scrapling.core._types import Dict, Optional from .custom import Response, StatusText +from scrapling.core._types import Dict, Optional + +__CHARSET_RE__ = re_compile(r"charset=([\w-]+)") class ResponseFactory: @@ -18,9 +23,19 @@ class ResponseFactory: """ @classmethod - def _process_response_history( - cls, first_response: SyncResponse, parser_arguments: Dict - ) -> list[Response]: + @lru_cache(maxsize=16) + def __extract_browser_encoding(cls, content_type: str | None) -> Optional[str]: + """Extract browser encoding from headers. + Ex: from header "content-type: text/html; charset=utf-8" -> "utf-8 + """ + if content_type: + # Because Playwright can't do that by themselves like all libraries for some reason :3 + match = __CHARSET_RE__.search(content_type) + return match.group(1) if match else None + return None + + @classmethod + def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]: """Process response history to build a list of `Response` objects""" history = [] current_request = first_response.request.redirected_from @@ -32,24 +47,23 @@ class ResponseFactory: history.insert( 0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - content="", - status=current_response.status if current_response else 301, - reason=( - current_response.status_text - or StatusText.get(current_response.status) - ) - if current_response - else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") - or "utf-8", - cookies=tuple(), - headers=current_response.all_headers() - if current_response - else {}, - request_headers=current_request.all_headers(), - **parser_arguments, + **{ + "url": current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + "content": "", + "status": current_response.status if current_response else 301, + "reason": (current_response.status_text or StatusText.get(current_response.status)) + if current_response + else StatusText.get(301), + "encoding": cls.__extract_browser_encoding( + current_response.headers.get("content-type", "") + ) + or "utf-8", + "cookies": tuple(), + "headers": current_response.all_headers() if current_response else {}, + "request_headers": current_request.all_headers(), + **parser_arguments, + } ), ) except Exception as e: # pragma: no cover @@ -93,14 +107,11 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - # This will be parsed inside `Response` encoding = ( - final_response.headers.get("content-type", "") or "utf-8" + cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" ) # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = final_response.status_text or StatusText.get( - final_response.status - ) + status_text = final_response.status_text or StatusText.get(final_response.status) history = cls._process_response_history(first_response, parser_arguments) try: @@ -110,16 +121,18 @@ class ResponseFactory: page_content = "" return Response( - url=page.url, - content=page_content, - status=final_response.status, - reason=status_text, - encoding=encoding, - cookies=tuple(dict(cookie) for cookie in page.context.cookies()), - headers=first_response.all_headers(), - request_headers=first_response.request.all_headers(), - history=history, - **parser_arguments, + **{ + "url": page.url, + "content": page_content, + "status": final_response.status, + "reason": status_text, + "encoding": encoding, + "cookies": tuple(dict(cookie) for cookie in page.context.cookies()), + "headers": first_response.all_headers(), + "request_headers": first_response.request.all_headers(), + "history": history, + **parser_arguments, + } ) @classmethod @@ -137,24 +150,23 @@ class ResponseFactory: history.insert( 0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - content="", - status=current_response.status if current_response else 301, - reason=( - current_response.status_text - or StatusText.get(current_response.status) - ) - if current_response - else StatusText.get(301), - encoding=current_response.headers.get("content-type", "") - or "utf-8", - cookies=tuple(), - headers=await current_response.all_headers() - if current_response - else {}, - request_headers=await current_request.all_headers(), - **parser_arguments, + **{ + "url": current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + "content": "", + "status": current_response.status if current_response else 301, + "reason": (current_response.status_text or StatusText.get(current_response.status)) + if current_response + else StatusText.get(301), + "encoding": cls.__extract_browser_encoding( + current_response.headers.get("content-type", "") + ) + or "utf-8", + "cookies": tuple(), + "headers": await current_response.all_headers() if current_response else {}, + "request_headers": await current_request.all_headers(), + **parser_arguments, + } ), ) except Exception as e: # pragma: no cover @@ -198,18 +210,13 @@ class ResponseFactory: if not final_response: raise ValueError("Failed to get a response from the page") - # This will be parsed inside `Response` encoding = ( - final_response.headers.get("content-type", "") or "utf-8" + cls.__extract_browser_encoding(final_response.headers.get("content-type", "")) or "utf-8" ) # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = final_response.status_text or StatusText.get( - final_response.status - ) + status_text = final_response.status_text or StatusText.get(final_response.status) - history = await cls._async_process_response_history( - first_response, parser_arguments - ) + history = await cls._async_process_response_history(first_response, parser_arguments) try: page_content = await page.content() except Exception as e: # pragma: no cover @@ -217,16 +224,18 @@ class ResponseFactory: page_content = "" return Response( - url=page.url, - content=page_content, - status=final_response.status, - reason=status_text, - encoding=encoding, - cookies=tuple(dict(cookie) for cookie in await page.context.cookies()), - headers=await first_response.all_headers(), - request_headers=await first_response.request.all_headers(), - history=history, - **parser_arguments, + **{ + "url": page.url, + "content": page_content, + "status": final_response.status, + "reason": status_text, + "encoding": encoding, + "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()), + "headers": await first_response.all_headers(), + "request_headers": await first_response.request.all_headers(), + "history": history, + **parser_arguments, + } ) @staticmethod @@ -238,17 +247,17 @@ class ResponseFactory: :return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ return Response( - url=response.url, - content=response.content - if isinstance(response.content, bytes) - else response.content.encode(), - status=response.status_code, - reason=response.reason, - encoding=response.encoding or "utf-8", - cookies=dict(response.cookies), - headers=dict(response.headers), - request_headers=dict(response.request.headers), - method=response.request.method, - history=response.history, # https://github.com/lexiforest/curl_cffi/issues/82 - **parser_arguments, + **{ + "url": response.url, + "content": response.content, + "status": response.status_code, + "reason": response.reason, + "encoding": response.encoding or "utf-8", + "cookies": dict(response.cookies), + "headers": dict(response.headers), + "request_headers": dict(response.request.headers), + "method": response.request.method, + "history": response.history, # https://github.com/lexiforest/curl_cffi/issues/82 + **parser_arguments, + } ) diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 79b52d1..774eec7 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -2,8 +2,9 @@ Functions related to custom types or type checking """ -from email.message import Message +from functools import lru_cache +from scrapling.core.utils import log from scrapling.core._types import ( Any, Dict, @@ -12,89 +13,9 @@ from scrapling.core._types import ( Tuple, ) from scrapling.core.custom_types import MappingProxyType -from scrapling.core.utils import log, lru_cache from scrapling.parser import Selector, SQLiteStorageSystem -class ResponseEncoding: - __DEFAULT_ENCODING = "utf-8" - __ISO_8859_1_CONTENT_TYPES = { - "text/plain", - "text/html", - "text/css", - "text/javascript", - } - - @classmethod - @lru_cache(maxsize=128) - def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]: - """Parse content type and parameters from a content-type header value. - - Uses `email.message.Message` for robust header parsing according to RFC 2045. - - :param header_value: Raw content-type header string - :return: Tuple of (content_type, parameters_dict) - """ - # Create a Message object and set the Content-Type header then get the content type and parameters - msg = Message() - msg["content-type"] = header_value - - content_type = msg.get_content_type() - params = dict(msg.get_params(failobj=[])) - - # Remove the content-type from params if present somehow - params.pop("content-type", None) - - return content_type, params - - @classmethod - @lru_cache(maxsize=128) - def get_value( - cls, content_type: Optional[str], text: Optional[str] = "test" - ) -> str: - """Determine the appropriate character encoding from a content-type header. - - The encoding is determined by these rules in order: - 1. If no content-type is provided, use UTF-8 - 2. If charset parameter is present, use that encoding - 3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec - 4. If content-type is application/json, use UTF-8 per RFC 4627 - 5. Default to UTF-8 if nothing else matches - - :param content_type: Content-Type header value or None - :param text: A text to test the encoding on it - :return: String naming the character encoding - """ - if not content_type: - return cls.__DEFAULT_ENCODING - - try: - encoding = None - content_type, params = cls.__parse_content_type(content_type) - - # First check for explicit charset parameter - if "charset" in params: - encoding = params["charset"].strip("'\"") - - # Apply content-type specific rules - elif content_type in cls.__ISO_8859_1_CONTENT_TYPES: - encoding = "ISO-8859-1" - - elif content_type == "application/json": - encoding = cls.__DEFAULT_ENCODING - - if encoding: - _ = text.encode( - encoding - ) # Validate encoding and validate it can encode the given text - return encoding - - return cls.__DEFAULT_ENCODING - - except (ValueError, LookupError, UnicodeEncodeError): - return cls.__DEFAULT_ENCODING - - class Response(Selector): """This class is returned by all engines as a way to unify response type between different libraries.""" @@ -119,9 +40,6 @@ class Response(Selector): self.headers = headers self.request_headers = request_headers self.history = history or [] - encoding = ResponseEncoding.get_value( - encoding, content.decode("utf-8") if isinstance(content, bytes) else content - ) super().__init__( content=content, url=adaptive_domain or url, @@ -129,9 +47,7 @@ class Response(Selector): **selector_config, ) # For easier debugging while working from a Python shell - log.info( - f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})" - ) + log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})") class BaseFetcher: @@ -190,18 +106,12 @@ class BaseFetcher: setattr(cls, key, value) else: # Yup, no fun allowed LOL - raise AttributeError( - f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?' - ) + raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') else: - raise ValueError( - f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?' - ) + raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') if not kwargs: - raise AttributeError( - f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?" - ) + raise AttributeError(f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?") @classmethod def _generate_parser_arguments(cls) -> Dict: @@ -217,9 +127,7 @@ class BaseFetcher: ) if cls.adaptive_domain: if not isinstance(cls.adaptive_domain, str): - log.warning( - '[Ignored] The argument "adaptive_domain" must be of string type' - ) + log.warning('[Ignored] The argument "adaptive_domain" must be of string type') else: parser_arguments.update({"adaptive_domain": cls.adaptive_domain}) diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 4ca8e38..bd836e7 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -2,13 +2,13 @@ Functions related to generating headers and fingerprints generally """ +from functools import lru_cache from platform import system as platform_system from tldextract import extract from browserforge.headers import Browser, HeaderGenerator from scrapling.core._types import Dict, Optional -from scrapling.core.utils import lru_cache __OS_NAME__ = platform_system() @@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]: "Linux": "linux", "Darwin": "macos", "Windows": "windows", - # For the future? because why not? - "iOS": "ios", }.get(__OS_NAME__) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index ab91174..f2f445c 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -30,9 +30,7 @@ def intercept_route(route: Route): :return: PlayWright `Route` object """ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: - log.debug( - f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"' - ) + log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') route.abort() else: route.continue_() @@ -45,17 +43,13 @@ async def async_intercept_route(route: async_Route): :return: PlayWright `Route` object """ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: - log.debug( - f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"' - ) + log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') await route.abort() else: await route.continue_() -def construct_proxy_dict( - proxy_string: str | Dict[str, str], as_tuple=False -) -> Optional[Dict | Tuple]: +def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]: """Validate a proxy and return it in the acceptable format for Playwright Reference: https://playwright.dev/python/docs/network#http-proxy @@ -65,10 +59,7 @@ def construct_proxy_dict( """ if isinstance(proxy_string, str): proxy = urlparse(proxy_string) - if ( - proxy.scheme not in ("http", "https", "socks4", "socks5") - or not proxy.hostname - ): + if proxy.scheme not in ("http", "https", "socks4", "socks5") or not proxy.hostname: raise ValueError("Invalid proxy string!") try: @@ -95,51 +86,6 @@ def construct_proxy_dict( return None -def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: - """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists - - :param cdp_url: The target URL. - :param query_params: A dictionary of the parameters to add. - :return: The new CDP URL. - """ - try: - # Validate the base URL structure - parsed = urlparse(cdp_url) - - # Check scheme - if parsed.scheme not in ("ws", "wss"): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") - - # Validate hostname and port - if not parsed.netloc: - raise ValueError("Invalid hostname for the CDP URL") - - try: - # Checking if the port is valid (if available) - _ = parsed.port - except ValueError: - # urlparse will raise `ValueError` if the port can't be casted to integer - raise ValueError("Invalid port for the CDP URL") - - # Ensure the path starts with / - path = parsed.path - if not path.startswith("/"): - path = "/" + path - - # Reconstruct the base URL with validated parts - validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" - - # Add query parameters - if query_params: - query_string = urlencode(query_params) - return f"{validated_base}?{query_string}" - - return validated_base - - except Exception as e: - raise ValueError(f"Invalid CDP URL: {str(e)}") - - @lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index c1ce8f7..fbb2b28 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -6,16 +6,18 @@ from scrapling.core._types import ( SelectorWaitStates, Iterable, ) -from scrapling.engines import ( +from scrapling.engines.static import ( FetcherSession, - StealthySession, - AsyncStealthySession, - DynamicSession, - AsyncDynamicSession, FetcherClient as _FetcherClient, AsyncFetcherClient as _AsyncFetcherClient, ) -from scrapling.engines.toolbelt import BaseFetcher, Response +from scrapling.engines._browsers import ( + DynamicSession, + StealthySession, + AsyncDynamicSession, + AsyncStealthySession, +) +from scrapling.engines.toolbelt.custom import BaseFetcher, Response __FetcherClientInstance__ = _FetcherClient() __AsyncFetcherClientInstance__ = _AsyncFetcherClient() @@ -56,6 +58,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -92,11 +95,12 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -112,13 +116,10 @@ class StealthyFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") with StealthySession( wait=wait, - max_pages=1, proxy=proxy, geoip=geoip, addons=addons, @@ -126,6 +127,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -155,6 +157,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -191,11 +194,12 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. @@ -211,9 +215,7 @@ class StealthyFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") async with AsyncStealthySession( wait=wait, @@ -225,6 +227,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -285,6 +288,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -298,9 +302,10 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -319,9 +324,7 @@ class DynamicFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - raise ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") with DynamicSession( wait=wait, @@ -332,6 +335,7 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, real_chrome=real_chrome, page_action=page_action, @@ -371,6 +375,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -384,9 +389,10 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -405,12 +411,11 @@ class DynamicFetcher(BaseFetcher): if not custom_config: custom_config = {} elif not isinstance(custom_config, dict): - raise ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) + raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") async with AsyncDynamicSession( wait=wait, + max_pages=1, proxy=proxy, locale=locale, timeout=timeout, @@ -418,8 +423,8 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, - max_pages=1, real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, diff --git a/scrapling/parser.py b/scrapling/parser.py index 730f490..409e09d 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,12 +1,11 @@ -from pathlib import Path import re +from pathlib import Path from inspect import signature -from difflib import SequenceMatcher from urllib.parse import urljoin +from difflib import SequenceMatcher -from cssselect import SelectorError, SelectorSyntaxError -from cssselect import parse as split_selectors from lxml.html import HtmlElement, HtmlMixin, HTMLParser +from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors from lxml.etree import ( XPath, tostring, @@ -75,7 +74,7 @@ class Selector(SelectorsGeneration): self, content: Optional[str | bytes] = None, url: Optional[str] = None, - encoding: str = "utf8", + encoding: str = "utf-8", huge_tree: bool = True, root: Optional[HtmlElement] = None, keep_comments: Optional[bool] = False, @@ -110,22 +109,16 @@ class Selector(SelectorsGeneration): If empty, default values will be used. """ if root is None and content is None: - raise ValueError( - "Selector class needs HTML content, or root arguments to work" - ) + raise ValueError("Selector class needs HTML content, or root arguments to work") self.__text = None if root is None: if isinstance(content, str): - body = ( - content.strip().replace("\x00", "").encode(encoding) or b"" - ) + body = content.strip().replace("\x00", "").encode(encoding) or b"" elif isinstance(content, bytes): - body = content.replace(b"\x00", b"").strip() + body = content.replace(b"\x00", b"") else: - raise TypeError( - f"content argument must be str or bytes, got {type(content)}" - ) + raise TypeError(f"content argument must be str or bytes, got {type(content)}") # https://lxml.de/api/lxml.etree.HTMLParser-class.html parser = HTMLParser( @@ -139,8 +132,7 @@ class Selector(SelectorsGeneration): strip_cdata=(not keep_cdata), ) self._root = fromstring(body, parser=parser, base_url=url) - - self._raw_body = body.decode() + self._raw_body = content else: # All HTML types inherit from HtmlMixin so this to check for all at once @@ -165,16 +157,10 @@ class Selector(SelectorsGeneration): } if not hasattr(storage, "__wrapped__"): - raise ValueError( - "Storage class must be wrapped with lru_cache decorator, see docs for info" - ) + raise ValueError("Storage class must be wrapped with lru_cache decorator, see docs for info") - if not issubclass( - storage.__wrapped__, StorageSystemMixin - ): # pragma: no cover - raise ValueError( - "Storage system must be inherited from class `StorageSystemMixin`" - ) + if not issubclass(storage.__wrapped__, StorageSystemMixin): # pragma: no cover + raise ValueError("Storage system must be inherited from class `StorageSystemMixin`") self._storage = storage(**storage_args) @@ -239,9 +225,7 @@ class Selector(SelectorsGeneration): def __element_convertor(self, element: HtmlElement) -> "Selector": """Used internally to convert a single HtmlElement to Selector directly without checks""" - db_instance = ( - self._storage if (hasattr(self, "_storage") and self._storage) else None - ) + db_instance = self._storage if (hasattr(self, "_storage") and self._storage) else None return Selector( root=element, url=self.url, @@ -355,18 +339,19 @@ class Selector(SelectorsGeneration): @property def html_content(self) -> TextHandler: """Return the inner HTML code of the element""" - return TextHandler( - tostring(self._root, encoding="unicode", method="html", with_tail=False) - ) + return TextHandler(tostring(self._root, encoding=self.encoding, method="html", with_tail=False)) - body = html_content + @property + def body(self): + """Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests.""" + return self._raw_body def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" return TextHandler( tostring( self._root, - encoding="unicode", + encoding=self.encoding, pretty_print=True, method="html", with_tail=False, @@ -404,9 +389,7 @@ class Selector(SelectorsGeneration): def siblings(self) -> "Selectors": """Return other children of the current element's parent or empty list otherwise""" if self.parent: - return Selectors( - child for child in self.parent.children if child._root != self._root - ) + return Selectors(child for child in self.parent.children if child._root != self._root) return Selectors() def iterancestors(self) -> Generator["Selector", None, None]: @@ -519,9 +502,7 @@ class Selector(SelectorsGeneration): log.debug(f"Highest probability was {highest_probability}%") log.debug("Top 5 best matching elements are: ") for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]: - log.debug( - f"{percent} -> {self.__handle_elements(score_table[percent])}" - ) + log.debug(f"{percent} -> {self.__handle_elements(score_table[percent])}") if not selector_type: return score_table[highest_probability] @@ -658,9 +639,7 @@ class Selector(SelectorsGeneration): SelectorError, SelectorSyntaxError, ) as e: - raise SelectorSyntaxError( - f"Invalid CSS selector '{selector}': {str(e)}" - ) from e + raise SelectorSyntaxError(f"Invalid CSS selector '{selector}': {str(e)}") from e def xpath( self, @@ -702,9 +681,7 @@ class Selector(SelectorsGeneration): elif self.__adaptive_enabled and auto_save: self.save(elements[0], identifier or selector) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) elif self.__adaptive_enabled: if adaptive: element_data = self.retrieve(identifier or selector) @@ -713,9 +690,7 @@ class Selector(SelectorsGeneration): if elements is not None and auto_save: self.save(elements[0], identifier or selector) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) else: if adaptive: log.warning( @@ -726,9 +701,7 @@ class Selector(SelectorsGeneration): "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info." ) - return self.__handle_elements( - elements[0:1] if (_first_match and elements) else elements - ) + return self.__handle_elements(elements[0:1] if (_first_match and elements) else elements) except ( SelectorError, @@ -751,9 +724,7 @@ class Selector(SelectorsGeneration): """ if not args and not kwargs: - raise TypeError( - "You have to pass something to search with, like tag name(s), tag attributes, or both." - ) + raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.") attributes = dict() tags, patterns = set(), set() @@ -766,18 +737,11 @@ class Selector(SelectorsGeneration): elif type(arg) in (list, tuple, set): if not all(map(lambda x: isinstance(x, str), arg)): - raise TypeError( - "Nested Iterables are not accepted, only iterables of tag names are accepted" - ) + raise TypeError("Nested Iterables are not accepted, only iterables of tag names are accepted") tags.update(set(arg)) elif isinstance(arg, dict): - if not all( - [ - (isinstance(k, str) and isinstance(v, str)) - for k, v in arg.items() - ] - ): + if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in arg.items()]): raise TypeError( "Nested dictionaries are not accepted, only string keys and string values are accepted" ) @@ -795,13 +759,9 @@ class Selector(SelectorsGeneration): ) else: - raise TypeError( - f'Argument with type "{type(arg)}" is not accepted, please read the docs.' - ) + raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.') - if not all( - [(isinstance(k, str) and isinstance(v, str)) for k, v in kwargs.items()] - ): + if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in kwargs.items()]): raise TypeError("Only string values are accepted for arguments") for attribute_name, value in kwargs.items(): @@ -825,9 +785,7 @@ class Selector(SelectorsGeneration): if results: # From the results, get the ones that fulfill passed regex patterns for pattern in patterns: - results = results.filter( - lambda e: e.text.re(pattern, check_match=True) - ) + results = results.filter(lambda e: e.text.re(pattern, check_match=True)) # From the results, get the ones that fulfill passed functions for function in functions: @@ -858,9 +816,7 @@ class Selector(SelectorsGeneration): return element return None - def __calculate_similarity_score( - self, original: Dict, candidate: HtmlElement - ) -> float: + def __calculate_similarity_score(self, original: Dict, candidate: HtmlElement) -> float: """Used internally to calculate a score that shows how a candidate element similar to the original one :param original: The original element in the form of the dictionary generated from `element_to_dict` function @@ -877,15 +833,11 @@ class Selector(SelectorsGeneration): checks += 1 if original["text"]: - score += SequenceMatcher( - None, original["text"], candidate.get("text") or "" - ).ratio() # * 0.3 # 30% + score += SequenceMatcher(None, original["text"], candidate.get("text") or "").ratio() # * 0.3 # 30% checks += 1 # if both don't have attributes, it still counts for something! - score += self.__calculate_dict_diff( - original["attributes"], candidate["attributes"] - ) # * 0.3 # 30% + score += self.__calculate_dict_diff(original["attributes"], candidate["attributes"]) # * 0.3 # 30% checks += 1 # Separate similarity test for class, id, href,... this will help in full structural changes @@ -903,9 +855,7 @@ class Selector(SelectorsGeneration): ).ratio() # * 0.3 # 30% checks += 1 - score += SequenceMatcher( - None, original["path"], candidate["path"] - ).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["path"], candidate["path"]).ratio() # * 0.1 # 10% checks += 1 if original.get("parent_name"): @@ -944,14 +894,8 @@ class Selector(SelectorsGeneration): @staticmethod def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float: """Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries""" - score = ( - SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() - * 0.5 - ) - score += ( - SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() - * 0.5 - ) + score = SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() * 0.5 + score += SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() * 0.5 return score def save(self, element: Union["Selector", HtmlElement], identifier: str) -> None: @@ -992,7 +936,7 @@ class Selector(SelectorsGeneration): # Operations on text functions def json(self) -> Dict: """Return JSON response if the response is jsonable otherwise throws error""" - if self._raw_body: + if self._raw_body and isinstance(self._raw_body, str): return TextHandler(self._raw_body).json() elif self.text: return self.text.json() @@ -1031,9 +975,7 @@ class Selector(SelectorsGeneration): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it """ - return self.text.re_first( - regex, default, replace_entities, clean_match, case_sensitive - ) + return self.text.re_first(regex, default, replace_entities, clean_match, case_sensitive) @staticmethod def __get_attributes(element: HtmlElement, ignore_attributes: List | Tuple) -> Dict: @@ -1052,9 +994,7 @@ class Selector(SelectorsGeneration): """Calculate a score of how much these elements are alike and return True if the score is higher or equals the threshold""" candidate_attributes = ( - self.__get_attributes(candidate, ignore_attributes) - if ignore_attributes - else candidate.attrib + self.__get_attributes(candidate, ignore_attributes) if ignore_attributes else candidate.attrib ) score, checks = 0, 0 @@ -1116,11 +1056,7 @@ class Selector(SelectorsGeneration): similar_elements = list() current_depth = len(list(root.iterancestors())) - target_attrs = ( - self.__get_attributes(root, ignore_attributes) - if ignore_attributes - else root.attrib - ) + target_attrs = self.__get_attributes(root, ignore_attributes) if ignore_attributes else root.attrib path_parts = [self.tag] if (parent := root.getparent()) is not None: @@ -1129,9 +1065,7 @@ class Selector(SelectorsGeneration): path_parts.insert(0, grandparent.tag) xpath_path = "//{}".format("/".join(path_parts)) - potential_matches = root.xpath( - f"{xpath_path}[count(ancestor::*) = {current_depth}]" - ) + potential_matches = root.xpath(f"{xpath_path}[count(ancestor::*) = {current_depth}]") for potential_match in potential_matches: if potential_match != root and self.__are_alike( @@ -1275,12 +1209,7 @@ class Selectors(List[Selector]): :return: `Selectors` class. """ - results = [ - n.xpath( - selector, identifier or selector, False, auto_save, percentage, **kwargs - ) - for n in self - ] + results = [n.xpath(selector, identifier or selector, False, auto_save, percentage, **kwargs) for n in self] return self.__class__(flatten(results)) def css( @@ -1308,10 +1237,7 @@ class Selectors(List[Selector]): :return: `Selectors` class. """ - results = [ - n.css(selector, identifier or selector, False, auto_save, percentage) - for n in self - ] + results = [n.css(selector, identifier or selector, False, auto_save, percentage) for n in self] return self.__class__(flatten(results)) def re( @@ -1329,10 +1255,7 @@ class Selectors(List[Selector]): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it """ - results = [ - n.text.re(regex, replace_entities, clean_match, case_sensitive) - for n in self - ] + results = [n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self] return TextHandlers(flatten(results)) def re_first( diff --git a/setup.cfg b/setup.cfg index ae28f61..1cac942 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.1 +version = 0.3.2 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 348c1ce..cae9fa0 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -3,12 +3,24 @@ from click.testing import CliRunner from unittest.mock import patch, MagicMock import pytest_httpbin +from scrapling.parser import Selector from scrapling.cli import ( shell, mcp, get, post, put, delete, fetch, stealthy_fetch ) @pytest_httpbin.use_class_based_httpbin +def configure_selector_mock(): + """Helper function to create a properly configured Selector mock""" + mock_response = MagicMock(spec=Selector) + mock_response.body = "Test content" + mock_response.encoding = "utf-8" + mock_response.get_all_text.return_value = "Test content" + mock_response.css_first.return_value = mock_response + mock_response.css.return_value = [mock_response] + return mock_response + + class TestCLI: """Test CLI functionality""" @@ -45,136 +57,129 @@ class TestCLI: output_file = tmp_path / "output.md" with patch('scrapling.fetchers.Fetcher.get') as mock_get: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_response.status = 200 mock_get.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - get, - [html_url, str(output_file)] - ) - assert result.exit_code == 0 + result = runner.invoke( + get, + [html_url, str(output_file)] + ) + assert result.exit_code == 0 # Test with various options with patch('scrapling.fetchers.Fetcher.get') as mock_get: mock_get.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - get, - [ - html_url, - str(output_file), - '-H', 'User-Agent: Test', - '--cookies', 'session=abc123', - '--timeout', '60', - '--proxy', 'http://proxy:8080', - '-s', '.content', - '-p', 'page=1' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + get, + [ + html_url, + str(output_file), + '-H', 'User-Agent: Test', + '--cookies', 'session=abc123', + '--timeout', '60', + '--proxy', 'http://proxy:8080', + '-s', '.content', + '-p', 'page=1' + ] + ) + assert result.exit_code == 0 def test_extract_post_command(self, runner, tmp_path, html_url): """Test extract `post` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.post') as mock_post: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_post.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - post, - [ - html_url, - str(output_file), - '-d', 'key=value', - '-j', '{"data": "test"}' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + post, + [ + html_url, + str(output_file), + '-d', 'key=value', + '-j', '{"data": "test"}' + ] + ) + assert result.exit_code == 0 def test_extract_put_command(self, runner, tmp_path, html_url): """Test extract `put` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.put') as mock_put: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_put.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - put, - [ - html_url, - str(output_file), - '-d', 'key=value', - '-j', '{"data": "test"}' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + put, + [ + html_url, + str(output_file), + '-d', 'key=value', + '-j', '{"data": "test"}' + ] + ) + assert result.exit_code == 0 def test_extract_delete_command(self, runner, tmp_path, html_url): """Test extract `delete` command""" output_file = tmp_path / "output.html" with patch('scrapling.fetchers.Fetcher.delete') as mock_delete: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_delete.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - delete, - [ - html_url, - str(output_file) - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + delete, + [ + html_url, + str(output_file) + ] + ) + assert result.exit_code == 0 def test_extract_fetch_command(self, runner, tmp_path, html_url): """Test extract fetch command""" output_file = tmp_path / "output.txt" with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_fetch.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - fetch, - [ - html_url, - str(output_file), - '--headless', - '--stealth', - '--timeout', '60000' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + fetch, + [ + html_url, + str(output_file), + '--headless', + '--stealth', + '--timeout', '60000' + ] + ) + assert result.exit_code == 0 def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url): """Test extract fetch command""" output_file = tmp_path / "output.md" with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch: - mock_response = MagicMock() + mock_response = configure_selector_mock() mock_fetch.return_value = mock_response - with patch('scrapling.cli.Convertor.write_content_to_file'): - result = runner.invoke( - stealthy_fetch, - [ - html_url, - str(output_file), - '--headless', - '--css-selector', 'body', - '--timeout', '60000' - ] - ) - assert result.exit_code == 0 + result = runner.invoke( + stealthy_fetch, + [ + html_url, + str(output_file), + '--headless', + '--css-selector', 'body', + '--timeout', '60000' + ] + ) + assert result.exit_code == 0 def test_invalid_arguments(self, runner, html_url): """Test invalid arguments handling""" @@ -182,12 +187,8 @@ class TestCLI: result = runner.invoke(get) assert result.exit_code != 0 - # Invalid output file extension - with patch('scrapling.cli.Convertor.write_content_to_file') as mock_write: - mock_write.side_effect = ValueError("Unknown file type") - - _ = runner.invoke( - get, - [html_url, 'output.invalid'] - ) - # Should handle the error gracefully + _ = runner.invoke( + get, + [html_url, 'output.invalid'] + ) + # Should handle the error gracefully diff --git a/tests/fetchers/async/test_camoufox.py b/tests/fetchers/async/test_camoufox.py index 6a0700b..ffe6eac 100644 --- a/tests/fetchers/async/test_camoufox.py +++ b/tests/fetchers/async/test_camoufox.py @@ -1,3 +1,4 @@ +from playwright._impl._errors import TimeoutError import pytest import pytest_httpbin @@ -23,14 +24,9 @@ class TestStealthyFetcher: "basic_url": f"{url}/get", "html_url": f"{url}/html", "delayed_url": f"{url}/delay/10", # 10 Seconds delay response - "cookies_url": f"{url}/cookies/set/test/value", - "cloudflare_url": "https://nopecha.com/demo/cloudflare", # Interactive turnstile page + "cookies_url": f"{url}/cookies/set/test/value" } - async def test_cloudflare_fetch(self, fetcher, urls): - """Test if Cloudflare bypass is working""" - assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200 - async def test_basic_fetch(self, fetcher, urls): """Test doing a basic fetch request with multiple statuses""" assert (await fetcher.async_fetch(urls["status_200"])).status == 200 @@ -86,9 +82,3 @@ class TestStealthyFetcher: **kwargs ) assert response.status == 200 - - async def test_infinite_timeout(self, fetcher, urls): - """Test if infinite timeout breaks the code or not""" - assert ( - await fetcher.async_fetch(urls["delayed_url"], timeout=0) - ).status == 200 diff --git a/tests/fetchers/async/test_camoufox_session.py b/tests/fetchers/async/test_camoufox_session.py index a2e0075..2971488 100644 --- a/tests/fetchers/async/test_camoufox_session.py +++ b/tests/fetchers/async/test_camoufox_session.py @@ -4,7 +4,7 @@ import asyncio import pytest_httpbin -from scrapling.engines import AsyncStealthySession +from scrapling.engines._browsers import AsyncStealthySession @pytest_httpbin.use_class_based_httpbin diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py index 0d171ce..04106fa 100644 --- a/tests/fetchers/async/test_dynamic.py +++ b/tests/fetchers/async/test_dynamic.py @@ -90,9 +90,3 @@ class TestDynamicFetcherAsync: with pytest.raises(Exception): await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah") - - @pytest.mark.asyncio - async def test_infinite_timeout(self, fetcher, urls): - """Test if infinite timeout breaks the code or not""" - response = await fetcher.async_fetch(urls["delayed_url"], timeout=0) - assert response.status == 200 diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py index 24c6860..d7a4ea9 100644 --- a/tests/fetchers/async/test_dynamic_session.py +++ b/tests/fetchers/async/test_dynamic_session.py @@ -3,7 +3,7 @@ import asyncio import pytest_httpbin -from scrapling.engines import AsyncDynamicSession +from scrapling.engines._browsers import AsyncDynamicSession @pytest_httpbin.use_class_based_httpbin diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.py index 0207777..83c5f6c 100644 --- a/tests/fetchers/sync/test_camoufox.py +++ b/tests/fetchers/sync/test_camoufox.py @@ -22,11 +22,6 @@ class TestStealthyFetcher: self.html_url = f"{httpbin.url}/html" self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response self.cookies_url = f"{httpbin.url}/cookies/set/test/value" - self.cloudflare_url = "https://nopecha.com/demo/cloudflare" # Interactive turnstile page - - def test_cloudflare_fetch(self, fetcher): - """Test if Cloudflare bypass is working""" - assert fetcher.fetch(self.cloudflare_url, solve_cloudflare=True).status == 200 def test_basic_fetch(self, fetcher): """Test doing a basic fetch request with multiple statuses""" @@ -82,7 +77,3 @@ class TestStealthyFetcher: **kwargs ) assert response.status == 200 - - def test_infinite_timeout(self, fetcher): - """Test if infinite timeout breaks the code or not""" - assert fetcher.fetch(self.delayed_url, timeout=0).status == 200 diff --git a/tests/fetchers/sync/test_camoufox_session.py b/tests/fetchers/sync/test_camoufox_session.py index c062708..4ea6f44 100644 --- a/tests/fetchers/sync/test_camoufox_session.py +++ b/tests/fetchers/sync/test_camoufox_session.py @@ -53,7 +53,6 @@ class TestStealthySession: """Test if the session is created correctly""" with StealthySession( - max_pages=3, headless=True, block_images=True, disable_resources=True, @@ -63,7 +62,7 @@ class TestStealthySession: cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}], ) as session: - assert session.max_pages == 3 + assert session.max_pages == 1 assert session.headless is True assert session.block_images is True assert session.disable_resources is True diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index fe85bd3..e3ba3bc 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -24,8 +24,8 @@ class TestPageInfo: assert page_info.state == "busy" assert page_info.url == "https://example.com" - page_info.mark_ready() - assert page_info.state == "ready" + page_info.mark_finished() + assert page_info.state == "finished" assert page_info.url == "" page_info.mark_error() @@ -63,7 +63,6 @@ class TestPagePool: assert pool.max_pages == 5 assert pool.pages_count == 0 - assert pool.ready_count == 0 assert pool.busy_count == 0 def test_add_page(self): @@ -97,42 +96,13 @@ class TestPagePool: page1 = pool.add_page(Mock()) page2 = pool.add_page(Mock()) - # Mark one as busy - page1.mark_busy("https://example.com") + # Mark them as finished + page1.mark_finished() + page2.mark_finished() - # Should get the ready page - ready_page = pool.get_ready_page() - assert ready_page == page2 - - def test_get_ready_page_none_available(self): - """Test getting ready page when none available""" - pool = PagePool(max_pages=2) - - # Add pages and mark all as busy - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - page1.mark_busy("https://example1.com") - page2.mark_busy("https://example2.com") - - # Should return None - ready_page = pool.get_ready_page() - assert ready_page is None - - def test_page_counts(self): - """Test page count properties""" - pool = PagePool(max_pages=3) - - # Add pages with different states - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - page3 = pool.add_page(Mock()) - - page1.mark_busy("https://example.com") - page3.mark_error() - - assert pool.pages_count == 3 - assert pool.ready_count == 1 - assert pool.busy_count == 1 + # test + pool.close_all_finished_pages() + assert pool.pages_count == 0 def test_cleanup_error_pages(self): """Test cleaning up error pages""" @@ -140,7 +110,7 @@ class TestPagePool: # Add pages page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) + _ = pool.add_page(Mock()) page3 = pool.add_page(Mock()) # Mark some as error @@ -151,4 +121,4 @@ class TestPagePool: pool.cleanup_error_pages() - assert pool.pages_count == 1 # Only page2 should remain + assert pool.pages_count == 1 # Only 2 should remain diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py index 7ff0f18..1327db8 100644 --- a/tests/fetchers/test_response_handling.py +++ b/tests/fetchers/test_response_handling.py @@ -1,8 +1,7 @@ from unittest.mock import Mock from scrapling.parser import Selector -from scrapling.engines.toolbelt import ResponseFactory, Response -from scrapling.engines.toolbelt.custom import ResponseEncoding +from scrapling.engines.toolbelt.convertor import ResponseFactory, Response class TestResponseFactory: @@ -32,20 +31,6 @@ class TestResponseFactory: assert response.url == "https://example.com" assert isinstance(response, Response) - def test_response_encoding_edge_cases(self): - """Test response encoding handling""" - # Test various content types - test_cases = [ - (None, "utf-8"), - ("", "utf-8"), - ("text/html; charset=invalid", "utf-8"), - ("application/octet-stream", "utf-8"), - ] - - for content_type, expected in test_cases: - encoding = ResponseEncoding.get_value(content_type) - assert encoding == expected - def test_response_history_processing(self): """Test processing response history""" # Mock responses with redirects diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index b787d36..0028902 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,10 +1,9 @@ import pytest from pathlib import Path -from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response +from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, - construct_cdp_url, js_bypass_path ) from scrapling.engines.toolbelt.fingerprints import ( @@ -132,12 +131,6 @@ def status_map(): } -def test_parsing_content_type(content_type_map): - """Test if parsing different types of 'content-type' returns the expected result""" - for header_value, expected_encoding in content_type_map.items(): - assert ResponseEncoding.get_value(header_value) == expected_encoding - - def test_parsing_response_status(status_map): """Test if using different http responses' status codes returns the expected result""" for status_code, expected_status_text in status_map.items(): @@ -216,43 +209,6 @@ class TestConstructProxyDict: construct_proxy_dict({"invalid": "structure"}) -class TestConstructCdpUrl: - """Test CDP URL construction""" - - def test_basic_cdp_url(self): - """Test basic CDP URL""" - result = construct_cdp_url("ws://localhost:9222/devtools/browser") - assert result == "ws://localhost:9222/devtools/browser" - - def test_cdp_url_with_params(self): - """Test CDP URL with query parameters""" - params = {"timeout": "30000", "headless": "true"} - result = construct_cdp_url("ws://localhost:9222/devtools/browser", params) - - assert "timeout=30000" in result - assert "headless=true" in result - - def test_cdp_url_without_leading_slash(self): - """Test CDP URL without a leading slash in the path""" - with pytest.raises(ValueError): - construct_cdp_url("ws://localhost:9222devtools/browser") - - def test_invalid_cdp_scheme(self): - """Test invalid CDP URL scheme""" - with pytest.raises(ValueError): - construct_cdp_url("http://localhost:9222/devtools/browser") - - def test_invalid_cdp_netloc(self): - """Test invalid CDP URL network location""" - with pytest.raises(ValueError): - construct_cdp_url("ws:///devtools/browser") - - def test_malformed_cdp_url(self): - """Test malformed CDP URL""" - with pytest.raises(ValueError): - construct_cdp_url("not-a-url") - - class TestJsBypassPath: """Test JavaScript bypass path utility""" diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index 71552f9..3ac81cf 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -99,7 +99,7 @@ class TestAdvancedSelectors: keep_comments=False, keep_cdata=False ) - content = page.body + content = page.html_content assert "Comment" not in content def test_advanced_xpath_variables(self, complex_html):