From 145c03daffb8b7b3b2d25e78ee7d03f2e9e8d123 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 3 Nov 2024 01:04:02 +0200 Subject: [PATCH] Big structure changes (check commit description) - Moved most of the parser functions/files to the core package. - Converted tools file to a package and made separate files for similar functions. - Now all fetcher engines return a Response object - Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing. - I added a new custom fetcher so the user can create and use an engine. - More... --- README.md | 2 + scrapling/__init__.py | 6 +- scrapling/core/__init__.py | 0 scrapling/{ => core}/_types.py | 0 scrapling/{ => core}/custom_types.py | 4 +- scrapling/{ => core}/mixins.py | 0 scrapling/{ => core}/storage_adaptors.py | 4 +- scrapling/{ => core}/translator.py | 4 +- scrapling/{ => core}/utils.py | 2 +- scrapling/engines/__init__.py | 5 +- scrapling/engines/camo.py | 51 ++++-- scrapling/engines/constants.py | 108 +++++++++++++ scrapling/engines/pw.py | 97 +++++------- scrapling/engines/static.py | 29 +++- scrapling/engines/toolbelt/__init__.py | 18 +++ scrapling/engines/toolbelt/custom.py | 136 ++++++++++++++++ scrapling/engines/toolbelt/fingerprints.py | 65 ++++++++ scrapling/engines/toolbelt/navigation.py | 43 +++++ scrapling/engines/tools.py | 174 --------------------- scrapling/fetcher.py | 134 +++++++++------- scrapling/parser.py | 12 +- 21 files changed, 565 insertions(+), 329 deletions(-) create mode 100644 scrapling/core/__init__.py rename scrapling/{ => core}/_types.py (100%) rename scrapling/{ => core}/custom_types.py (98%) rename scrapling/{ => core}/mixins.py (100%) rename scrapling/{ => core}/storage_adaptors.py (98%) rename scrapling/{ => core}/translator.py (98%) rename scrapling/{ => core}/utils.py (98%) create mode 100644 scrapling/engines/constants.py create mode 100644 scrapling/engines/toolbelt/__init__.py create mode 100644 scrapling/engines/toolbelt/custom.py create mode 100644 scrapling/engines/toolbelt/fingerprints.py create mode 100644 scrapling/engines/toolbelt/navigation.py delete mode 100644 scrapling/engines/tools.py diff --git a/README.md b/README.md index 2fe34e3..5dcfd04 100644 --- a/README.md +++ b/README.md @@ -477,6 +477,8 @@ This project includes code adapted from: - [brotector](https://github.com/kaliiiiiiiiii/brotector) - [fakebrowser](https://github.com/kkoooqq/fakebrowser) - [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches) +- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright) +- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox) ## Known Issues - In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone. diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 72cb0bb..8323d50 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,11 +1,11 @@ # Declare top-level shortcuts -from scrapling.fetcher import Fetcher +from scrapling.fetcher import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher from scrapling.parser import Adaptor, Adaptors -from scrapling.custom_types import TextHandler, AttributesHandler +from scrapling.core.custom_types import TextHandler, AttributesHandler __author__ = "Karim Shoair (karim.shoair@pm.me)" __version__ = "0.2" __copyright__ = "Copyright (c) 2024 Karim Shoair" -__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler', 'Fetcher'] +__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher'] diff --git a/scrapling/core/__init__.py b/scrapling/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scrapling/_types.py b/scrapling/core/_types.py similarity index 100% rename from scrapling/_types.py rename to scrapling/core/_types.py diff --git a/scrapling/custom_types.py b/scrapling/core/custom_types.py similarity index 98% rename from scrapling/custom_types.py rename to scrapling/core/custom_types.py index 5ff3b5f..bffc36f 100644 --- a/scrapling/custom_types.py +++ b/scrapling/core/custom_types.py @@ -2,8 +2,8 @@ import re from types import MappingProxyType from collections.abc import Mapping -from scrapling.utils import _is_iterable, flatten -from scrapling._types import Dict, List, Union, Pattern +from scrapling.core.utils import _is_iterable, flatten +from scrapling.core._types import Dict, List, Union, Pattern from orjson import loads, dumps from w3lib.html import replace_entities as _replace_entities diff --git a/scrapling/mixins.py b/scrapling/core/mixins.py similarity index 100% rename from scrapling/mixins.py rename to scrapling/core/mixins.py diff --git a/scrapling/storage_adaptors.py b/scrapling/core/storage_adaptors.py similarity index 98% rename from scrapling/storage_adaptors.py rename to scrapling/core/storage_adaptors.py index d9018a3..675b46d 100644 --- a/scrapling/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -5,8 +5,8 @@ import threading from hashlib import sha256 from abc import ABC, abstractmethod -from scrapling._types import Dict, Optional, Union -from scrapling.utils import _StorageTools, cache +from scrapling.core._types import Dict, Optional, Union +from scrapling.core.utils import _StorageTools, cache from lxml import html from tldextract import extract as tld diff --git a/scrapling/translator.py b/scrapling/core/translator.py similarity index 98% rename from scrapling/translator.py rename to scrapling/core/translator.py index 77679a6..41f5811 100644 --- a/scrapling/translator.py +++ b/scrapling/core/translator.py @@ -9,8 +9,8 @@ which will be important in future releases but most importantly... import re from w3lib.html import HTML5_WHITESPACE -from scrapling.utils import cache -from scrapling._types import Any, Optional, Protocol, Self +from scrapling.core.utils import cache +from scrapling.core._types import Any, Optional, Protocol, Self from cssselect.xpath import ExpressionError from cssselect.xpath import XPathExpr as OriginalXPathExpr diff --git a/scrapling/utils.py b/scrapling/core/utils.py similarity index 98% rename from scrapling/utils.py rename to scrapling/core/utils.py index 5f8efc9..db5ef15 100644 --- a/scrapling/utils.py +++ b/scrapling/core/utils.py @@ -4,7 +4,7 @@ from itertools import chain # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache -from scrapling._types import Dict, Iterable, Any +from scrapling.core._types import Dict, Iterable, Any from lxml import html diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index 89b5d62..d91e20a 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,6 +1,7 @@ from .camo import CamoufoxEngine from .static import StaticEngine -from .pw import PlaywrightEngine, DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS -from .tools import check_if_engine_usable +from .pw import PlaywrightEngine +from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS +from .toolbelt import check_if_engine_usable __all__ = ['CamoufoxEngine', 'PlaywrightEngine'] diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 26464ab..eebdf51 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -1,26 +1,28 @@ import logging -from scrapling._types import Union, Callable, Optional +from scrapling.core._types import Union, Callable, Optional, Dict -from .tools import check_type_validity, get_os_name, generate_convincing_referer +from scrapling.engines.toolbelt import ( + Response, + do_nothing, + get_os_name, + check_type_validity, + generate_convincing_referer, +) from camoufox.sync_api import Camoufox -def _do_nothing(page): - # Anything - return page - - class CamoufoxEngine: def __init__( self, headless: Union[bool, str] = True, - block_images: Optional[bool] = True, + block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, - page_action: Callable = _do_nothing, + page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: str = 'attached', + adaptor_arguments: Dict = None ): self.headless = headless self.block_images = bool(block_images) @@ -30,23 +32,24 @@ class CamoufoxEngine: if callable(page_action): self.page_action = page_action else: - self.page_action = _do_nothing + self.page_action = do_nothing logging.error('[Ignored] Argument "page_action" must be callable') self.wait_selector = wait_selector self.wait_selector_state = wait_selector_state + self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {} - def fetch(self, url: str): + def fetch(self, url: str) -> Response: with Camoufox( headless=self.headless, - block_images=self.block_images, + block_images=self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful os=get_os_name(), block_webrtc=self.block_webrtc, ) as browser: page = browser.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) - page.goto(url, referer=generate_convincing_referer(url)) + res = page.goto(url, referer=generate_convincing_referer(url)) page.wait_for_load_state(state="load") page.wait_for_load_state(state="domcontentloaded") if self.network_idle: @@ -58,6 +61,24 @@ class CamoufoxEngine: waiter = page.locator(self.wait_selector) waiter.wait_for(state=self.wait_selector_state) - html = page.content() + content_type = res.headers.get('content-type', '') + # Parse charset from content-type + encoding = 'utf-8' # default encoding + if 'charset=' in content_type.lower(): + encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip() + + response = Response( + url=res.url, + text=res.text(), + content=res.body(), + status=res.status, + reason=res.status_text, + encoding=encoding, + cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, + headers=res.all_headers(), + request_headers=res.request.all_headers(), + adaptor_arguments=self.adaptor_arguments + ) page.close() - return html + + return response diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py new file mode 100644 index 0000000..245e5c0 --- /dev/null +++ b/scrapling/engines/constants.py @@ -0,0 +1,108 @@ +# Disable loading these resources for speed +DEFAULT_DISABLED_RESOURCES = [ + 'font', + 'image', + 'media', + 'beacon', + 'object', + 'imageset', + 'texttrack', + 'websocket', + 'csp_report', + 'stylesheet', +] + +DEFAULT_STEALTH_FLAGS = [ + # Explanation: https://peter.sh/experiments/chromium-command-line-switches/ + # Generally this will make the browser faster and less detectable + '--no-pings', + '--incognito', + '--test-type', + '--lang=en-US', + '--mute-audio', + '--no-first-run', + '--disable-sync', + '--hide-scrollbars', + '--disable-logging', + '--start-maximized', # For headless check bypass + '--enable-async-dns', + '--disable-breakpad', + '--disable-infobars', + '--accept-lang=en-US', + '--use-mock-keychain', + '--disable-translate', + '--disable-extensions', + '--disable-voice-input', + '--window-position=0,0', + '--disable-wake-on-wifi', + '--ignore-gpu-blocklist', + '--enable-tcp-fast-open', + '--enable-web-bluetooth', + '--disable-hang-monitor', + '--password-store=basic', + '--disable-cloud-import', + '--disable-default-apps', + '--disable-print-preview', + '--disable-dev-shm-usage', + '--disable-popup-blocking', + '--metrics-recording-only', + '--disable-crash-reporter', + '--disable-partial-raster', + '--disable-gesture-typing', + '--disable-checker-imaging', + '--disable-prompt-on-repost', + '--force-color-profile=srgb', + '--font-render-hinting=none', + '--no-default-browser-check', + '--aggressive-cache-discard', + '--disable-component-update', + '--disable-cookie-encryption', + '--disable-domain-reliability', + '--disable-threaded-animation', + '--disable-threaded-scrolling', + # '--disable-reading-from-canvas', # For Firefox + '--enable-simple-cache-backend', + '--disable-background-networking', + '--disable-session-crashed-bubble', + '--enable-surface-synchronization', + '--disable-image-animation-resync', + '--disable-renderer-backgrounding', + '--disable-ipc-flooding-protection', + '--prerender-from-omnibox=disabled', + '--safebrowsing-disable-auto-update', + '--disable-offer-upload-credit-cards', + '--disable-features=site-per-process', + '--disable-background-timer-throttling', + '--disable-new-content-rendering-timeout', + '--run-all-compositor-stages-before-draw', + '--disable-client-side-phishing-detection', + '--disable-backgrounding-occluded-windows', + '--disable-layer-tree-host-memory-pressure', + '--autoplay-policy=no-user-gesture-required', + '--disable-offer-store-unmasked-wallet-cards', + '--disable-blink-features=AutomationControlled', + '--webrtc-ip-handling-policy=disable_non_proxied_udp', + '--disable-component-extensions-with-background-pages', + '--force-webrtc-ip-handling-policy=disable_non_proxied_udp', + '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance', + '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4', + '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees', +] + +# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container +NSTBROWSER_DEFAULT_QUERY = { + "once": True, + "headless": True, + "autoClose": True, + "fingerprint": { + "flags": { + "timezone": "BasedOnIp", + "screen": "Custom" + }, + "platform": 'linux', # support: windows, mac, linux + "kernel": 'chromium', # only support: chromium + "kernelMilestone": '128', + "hardwareConcurrency": 8, + "deviceMemory": 8, + }, +} diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index db353fa..4d628c3 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -1,43 +1,17 @@ import json import logging -from scrapling._types import Union, Callable, Optional, List, Dict +from scrapling.core._types import Union, Callable, Optional, List, Dict -from .tools import check_type_validity, generate_headers, js_bypass_path, construct_websocket_url, generate_convincing_referer - -# Disable loading these resources for speed -DEFAULT_DISABLED_RESOURCES = ['beacon', 'csp_report', 'font', 'image', 'imageset', 'media', 'object', 'texttrack', 'stylesheet', 'websocket'] -DEFAULT_STEALTH_FLAGS = [ - # Explanation: https://peter.sh/experiments/chromium-command-line-switches/ - # Generally this will make the browser faster and less detectable - '--incognito', '--accept-lang=en-US', '--lang=en-US', '--no-pings', '--mute-audio', '--no-first-run', '--no-default-browser-check', '--disable-cloud-import', - '--disable-gesture-typing', '--disable-offer-store-unmasked-wallet-cards', '--disable-offer-upload-credit-cards', '--disable-print-preview', '--disable-voice-input', - '--disable-wake-on-wifi', '--disable-cookie-encryption', '--ignore-gpu-blocklist', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open', - '--prerender-from-omnibox=disabled', '--enable-web-bluetooth', '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees', - '--aggressive-cache-discard', '--disable-ipc-flooding-protection', '--disable-blink-features=AutomationControlled', '--test-type', - '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance', - '--disable-breakpad', '--disable-component-update', '--disable-domain-reliability', '--disable-sync', '--disable-client-side-phishing-detection', - '--disable-hang-monitor', '--disable-popup-blocking', '--disable-prompt-on-repost', '--metrics-recording-only', '--safebrowsing-disable-auto-update', '--password-store=basic', - '--autoplay-policy=no-user-gesture-required', '--use-mock-keychain', '--force-webrtc-ip-handling-policy=disable_non_proxied_udp', - '--webrtc-ip-handling-policy=disable_non_proxied_udp', '--disable-session-crashed-bubble', '--disable-crash-reporter', '--disable-dev-shm-usage', '--force-color-profile=srgb', - '--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-infobars', - '--hide-scrollbars', '--disable-renderer-backgrounding', '--font-render-hinting=none', '--disable-logging', '--enable-surface-synchronization', - '--run-all-compositor-stages-before-draw', '--disable-threaded-animation', '--disable-threaded-scrolling', '--disable-checker-imaging', - '--disable-new-content-rendering-timeout', '--disable-image-animation-resync', '--disable-partial-raster', - '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4', - '--disable-layer-tree-host-memory-pressure', - '--window-position=0,0', - '--disable-features=site-per-process', - '--disable-default-apps', - '--disable-component-extensions-with-background-pages', - '--disable-extensions', - # "--disable-reading-from-canvas", # For Firefox - '--start-maximized' # For headless check bypass -] - - -def _do_nothing(page): - # Anything - return page +from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY +from scrapling.engines.toolbelt import ( + Response, + do_nothing, + js_bypass_path, + generate_headers, + check_type_validity, + construct_websocket_url, + generate_convincing_referer, +) class PlaywrightEngine: @@ -47,7 +21,7 @@ class PlaywrightEngine: useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, - page_action: Callable = _do_nothing, + page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached', stealth: bool = False, @@ -56,6 +30,7 @@ class PlaywrightEngine: cdp_url: Optional[str] = None, nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, + adaptor_arguments: Dict = None ): self.headless = headless self.disable_resources = disable_resources @@ -69,13 +44,14 @@ class PlaywrightEngine: if callable(page_action): self.page_action = page_action else: - self.page_action = _do_nothing + self.page_action = do_nothing logging.error('[Ignored] Argument "page_action" must be callable') self.wait_selector = wait_selector self.wait_selector_state = wait_selector_state self.nstbrowser_mode = bool(nstbrowser_mode) self.nstbrowser_config = nstbrowser_config + self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {} def _cdp_url_logic(self, flags: Optional[dict] = None): cdp_url = self.cdp_url @@ -83,23 +59,7 @@ class PlaywrightEngine: if self.nstbrowser_config and type(self.nstbrowser_config) is Dict: config = self.nstbrowser_config else: - # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container - query = { - "once": True, - "headless": True, - "autoClose": True, - "fingerprint": { - "flags": { - "timezone": "BasedOnIp", - "screen": "Custom" - }, - "platform": 'linux', # support: windows, mac, linux - "kernel": 'chromium', # only support: chromium - "kernelMilestone": '128', - "hardwareConcurrency": 8, - "deviceMemory": 8, - }, - } + query = NSTBROWSER_DEFAULT_QUERY.copy() if flags: query.update({ "args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary @@ -113,7 +73,7 @@ class PlaywrightEngine: return cdp_url - def fetch(self, url): + def fetch(self, url) -> Response: if not self.stealth: from playwright.sync_api import sync_playwright else: @@ -192,7 +152,7 @@ class PlaywrightEngine: page.add_init_script(path=js_bypass_path('screen_props.js')) page.add_init_script(path=js_bypass_path('playwright_fingerprint.js')) - page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None) + res = page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None) page.wait_for_load_state(state="load") page.wait_for_load_state(state="domcontentloaded") if self.network_idle: @@ -204,6 +164,23 @@ class PlaywrightEngine: waiter = page.locator(self.wait_selector) waiter.wait_for(state=self.wait_selector_state) - html = page.content() + content_type = res.headers.get('content-type', '') + # Parse charset from content-type + encoding = 'utf-8' # default encoding + if 'charset=' in content_type.lower(): + encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip() + + response = Response( + url=res.url, + text=res.text(), + content=res.body(), + status=res.status, + reason=res.status_text, + encoding=encoding, + cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, + headers=res.all_headers(), + request_headers=res.request.all_headers(), + adaptor_arguments=self.adaptor_arguments + ) page.close() - return html + return response diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index d4110e7..273e14b 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,9 +1,10 @@ import logging -from scrapling._types import Union, Optional, Dict -from .tools import generate_convincing_referer, generate_headers +from scrapling.core._types import Union, Optional, Dict +from .toolbelt import Response, generate_convincing_referer, generate_headers import httpx +from httpx._models import Response as httpxResponse class StaticEngine: @@ -11,10 +12,12 @@ class StaticEngine: self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, + adaptor_arguments: Dict = None ): self.timeout = timeout self.follow_redirects = bool(follow_redirects) self._extra_headers = generate_headers(browser_mode=False) + self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {} @staticmethod def _headers_job(headers, url, stealth): @@ -32,22 +35,36 @@ class StaticEngine: return headers + def _prepare_response(self, response: httpxResponse): + return Response( + url=str(response.url), + text=response.text, + content=response.content, + status=response.status_code, + reason=response.reason_phrase, + encoding=response.encoding or 'utf-8', + cookies=dict(response.cookies), + headers=dict(response.headers), + request_headers=response.request.headers, + adaptor_arguments=self.adaptor_arguments + ) + def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict): headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers) request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) - return request.text + return self._prepare_response(request) def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict): headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers) request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) - return request.text + return self._prepare_response(request) def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict): headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers) request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) - return request.text + return self._prepare_response(request) def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict): headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers) request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) - return request.text + return self._prepare_response(request) diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py new file mode 100644 index 0000000..3aa0aed --- /dev/null +++ b/scrapling/engines/toolbelt/__init__.py @@ -0,0 +1,18 @@ +from .fingerprints import ( + get_os_name, + generate_headers, + generate_convincing_referer, + generate_suitable_fingerprint, +) +from .custom import ( + Response, + do_nothing, + BaseFetcher, + get_variable_name, + check_type_validity, + check_if_engine_usable, +) +from .navigation import ( + js_bypass_path, + construct_websocket_url, +) diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py new file mode 100644 index 0000000..990244e --- /dev/null +++ b/scrapling/engines/toolbelt/custom.py @@ -0,0 +1,136 @@ +""" +Functions related to custom types or type checking +""" +import inspect +import logging +from dataclasses import dataclass, field + +from scrapling.parser import Adaptor, SQLiteStorageSystem +from scrapling.core._types import Any, List, Type, Union, Optional, Dict + + +@dataclass(frozen=True) +class Response: + url: str + text: str + content: bytes + status: int + reason: str + encoding: str = 'utf-8' # default encoding + cookies: Dict = field(default_factory=dict) + headers: Dict = field(default_factory=dict) + request_headers: Dict = field(default_factory=dict) + adaptor_arguments: Dict = field(default_factory=dict) + + @property + def adaptor(self): + if self.text: + return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments) + elif self.content: + return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments) + return None + + def __repr__(self): + return f'<{self.__class__.__name__} [{self.status} {self.reason}]>' + + +class BaseFetcher: + def __init__( + self, + # Adaptor class parameters + huge_tree: bool = True, + keep_comments: Optional[bool] = False, + auto_match: Optional[bool] = False, + storage: Any = SQLiteStorageSystem, + storage_args: Optional[Dict] = None, + debug: Optional[bool] = True, + ): + # I won't validate Adaptor's class parameters here again, I will leave it to be validated later + self.adaptor_arguments = dict( + huge_tree=huge_tree, + keep_comments=keep_comments, + auto_match=auto_match, + storage=storage, + storage_args=storage_args, + debug=debug, + ) + + +def check_if_engine_usable(engine): + # if isinstance(engine, type): + # raise TypeError("Expected an engine instance, not a class definition of the engine") + + if hasattr(engine, 'fetch'): + fetch_function = getattr(engine, "fetch") + if callable(fetch_function): + if len(inspect.signature(fetch_function).parameters) > 0: + return engine + else: + # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.") + raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.") + else: + # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'") + raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'") + else: + # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'") + raise TypeError("Invalid engine class! Engine class must have the method 'fetch'") + + +def get_variable_name(var: Any) -> Optional[str]: + """Get the name of a variable using global and local scopes. + :param var: The variable to find the name for + :return: The name of the variable if found, None otherwise + """ + for scope in [globals(), locals()]: + for name, value in scope.items(): + if value is var: + return name + return None + + +def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any: + """Check if a variable matches the specified type constraints. + :param variable: The variable to check + :param valid_types: List of valid types for the variable + :param default_value: Value to return if type check fails + :param critical: If True, raises TypeError instead of logging error + :param param_name: Optional parameter name for error messages + :return: The original variable if valid, default_value if invalid + :raise TypeError: If critical=True and type check fails + """ + # Use provided param_name or try to get it automatically + var_name = param_name or get_variable_name(variable) or "Unknown" + + # Convert valid_types to a list if None + valid_types = valid_types or [] + + # Handle None value + if variable is None: + if type(None) in valid_types: + return variable + error_msg = f'Argument "{var_name}" cannot be None' + if critical: + raise TypeError(error_msg) + logging.error(f'[Ignored] {error_msg}') + return default_value + + # If no valid_types specified and variable has a value, return it + if not valid_types: + return variable + + # Check if variable type matches any of the valid types + if not any(isinstance(variable, t) for t in valid_types): + type_names = [t.__name__ for t in valid_types] + error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}' + if critical: + raise TypeError(error_msg) + logging.error(f'[Ignored] {error_msg}') + return default_value + + return variable + + +# Pew Pew +def do_nothing(page): + # Just works as a filler for `page_action` argument in browser engines + return page diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py new file mode 100644 index 0000000..3969b0a --- /dev/null +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -0,0 +1,65 @@ +""" +Functions related to generating headers and fingerprints generally +""" + +import platform + +from tldextract import extract +from browserforge.fingerprints import FingerprintGenerator +from browserforge.headers import HeaderGenerator, Browser + + +def generate_convincing_referer(url): + """ + Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website + + >>> generate_convincing_referer('https://www.somewebsite.com/blah') + 'https://www.google.com/search?q=somewebsite' + + :param url: The URL you are about to fetch. + :return: + """ + website_name = extract(url).domain + return f'https://www.google.com/search?q={website_name}' + + +def get_os_name(): + # Get the OS name in the same format needed for browserforge + os_name = platform.system() + return { + 'Linux': 'linux', + 'Darwin': 'macos', + 'Windows': 'windows', + # For the future? because why not + 'iOS': 'ios', + }.get(os_name) + + +def generate_suitable_fingerprint(): + # This would be for Browserforge playwright injector + os_name = get_os_name() + return FingerprintGenerator( + browser=[Browser(name='chrome', min_version=128)], + os=os_name, # None is ignored + device='desktop' + ).generate() + + +def generate_headers(browser_mode=False): + if browser_mode: + # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using + # So we don't raise any inconsistency red flags while websites fingerprinting us + os_name = get_os_name() + return HeaderGenerator( + browser=[Browser(name='chrome', min_version=128)], + os=os_name, # None is ignored + device='desktop' + ).generate() + else: + # Here it's used for normal requests that aren't done through browsers so we can take it lightly + browsers = [ + Browser(name='chrome', min_version=120), + Browser(name='firefox', min_version=120), + Browser(name='edge', min_version=120), + ] + return HeaderGenerator(browser=browsers, device='desktop').generate() diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py new file mode 100644 index 0000000..bf99cb2 --- /dev/null +++ b/scrapling/engines/toolbelt/navigation.py @@ -0,0 +1,43 @@ +""" +Functions related to files and URLs +""" + +import os +from urllib.parse import urlparse, urlencode + + +def construct_websocket_url(base_url, query_params): + # Validate the base URL structure + try: + parsed = urlparse(base_url) + + # Check scheme + if parsed.scheme not in ('ws', 'wss'): + raise ValueError("URL must use 'ws://' or 'wss://' scheme") + + # Validate hostname and port + if not parsed.netloc: + raise ValueError("Invalid hostname") + + # Ensure path starts with / + path = parsed.path + if not path.startswith('/'): + path = '/' + path + + # Reconstruct the base URL with validated parts + validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" + + # Add query parameters + if query_params: + query_string = urlencode(query_params) + return f"{validated_base}?{query_string}" + + return validated_base + + except Exception as e: + raise ValueError(f"Invalid WebSocket URL: {str(e)}") + + +def js_bypass_path(filename): + current_directory = os.path.dirname(__file__) + return os.path.join(current_directory, 'bypasses', filename) diff --git a/scrapling/engines/tools.py b/scrapling/engines/tools.py deleted file mode 100644 index 3d4564e..0000000 --- a/scrapling/engines/tools.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -import logging -import inspect -import platform -from scrapling._types import Any, List, Type, Union, Optional -from urllib.parse import urlparse, urlencode - -from tldextract import extract -from browserforge.fingerprints import FingerprintGenerator -from browserforge.headers import HeaderGenerator, Browser - - -def generate_convincing_referer(url): - """ - Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website - - >>> generate_convincing_referer('https://www.somewebsite.com/blah') - 'https://www.google.com/search?q=somewebsite' - - :param url: The URL you are about to fetch. - :return: - """ - website_name = extract(url).domain - return f'https://www.google.com/search?q={website_name}' - - -def check_if_engine_usable(engine): - if isinstance(engine, type): - raise TypeError("Expected an engine instance, not a class definition of the engine") - - if hasattr(engine, 'fetch'): - fetch_function = getattr(engine, "fetch") - if callable(fetch_function): - if len(inspect.signature(fetch_function).parameters) > 0: - return engine - else: - raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.") - else: - raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'") - else: - raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'") - - -def construct_websocket_url(base_url, query_params): - # Validate the base URL structure - try: - parsed = urlparse(base_url) - - # Check scheme - if parsed.scheme not in ('ws', 'wss'): - raise ValueError("URL must use 'ws://' or 'wss://' scheme") - - # Validate hostname and port - if not parsed.netloc: - raise ValueError("Invalid hostname") - - # Ensure path starts with / - path = parsed.path - if not path.startswith('/'): - path = '/' + path - - # Reconstruct the base URL with validated parts - validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" - - # Add query parameters - if query_params: - query_string = urlencode(query_params) - return f"{validated_base}?{query_string}" - - return validated_base - - except Exception as e: - raise ValueError(f"Invalid WebSocket URL: {str(e)}") - - -def js_bypass_path(filename): - current_directory = os.path.dirname(__file__) - return os.path.join(current_directory, 'bypasses', filename) - - -def get_os_name(): - # Get the OS name in the same format needed for browserforge - os_name = platform.system() - return { - 'Linux': 'linux', - 'Darwin': 'macos', - 'Windows': 'windows', - # For the future? because why not - 'iOS': 'ios', - }.get(os_name) - - -def generate_suitable_fingerprint(): - # This would be for Browserforge playwright injector - os_name = get_os_name() - return FingerprintGenerator( - browser=[Browser(name='chrome', min_version=128)], - os=os_name, # None is ignored - device='desktop' - ).generate() - - -def generate_headers(browser_mode=False): - if browser_mode: - # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using - # So we don't raise any inconsistency red flags while websites fingerprinting us - os_name = get_os_name() - return HeaderGenerator( - browser=[Browser(name='chrome', min_version=128)], - os=os_name, # None is ignored - device='desktop' - ).generate() - else: - # Here it's used for normal requests that aren't done through browsers so we can take it lightly - browsers = [ - Browser(name='chrome', min_version=120), - Browser(name='firefox', min_version=120), - Browser(name='edge', min_version=120), - ] - return HeaderGenerator(browser=browsers, device='desktop').generate() - - -def get_variable_name(var: Any) -> Optional[str]: - """Get the name of a variable using global and local scopes. - :param var: The variable to find the name for - :return: The name of the variable if found, None otherwise - """ - for scope in [globals(), locals()]: - for name, value in scope.items(): - if value is var: - return name - return None - - -def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any: - """Check if a variable matches the specified type constraints. - :param variable: The variable to check - :param valid_types: List of valid types for the variable - :param default_value: Value to return if type check fails - :param critical: If True, raises TypeError instead of logging error - :param param_name: Optional parameter name for error messages - :return: The original variable if valid, default_value if invalid - :raise TypeError: If critical=True and type check fails - """ - # Use provided param_name or try to get it automatically - var_name = param_name or get_variable_name(variable) or "Unknown" - - # Convert valid_types to a list if None - valid_types = valid_types or [] - - # Handle None value - if variable is None: - if type(None) in valid_types: - return variable - error_msg = f'Argument "{var_name}" cannot be None' - if critical: - raise TypeError(error_msg) - logging.error(f'[Ignored] {error_msg}') - return default_value - - # If no valid_types specified and variable has a value, return it - if not valid_types: - return variable - - # Check if variable type matches any of the valid types - if not any(isinstance(variable, t) for t in valid_types): - type_names = [t.__name__ for t in valid_types] - error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}' - if critical: - raise TypeError(error_msg) - logging.error(f'[Ignored] {error_msg}') - return default_value - - return variable diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py index 065c46c..55c59b2 100644 --- a/scrapling/fetcher.py +++ b/scrapling/fetcher.py @@ -1,65 +1,87 @@ -from scrapling._types import Any, Dict, Optional, Union +from scrapling.core._types import Dict, Optional, Union, Callable, List -from scrapling.engines import CamoufoxEngine, StaticEngine, check_if_engine_usable -from scrapling.parser import Adaptor, SQLiteStorageSystem +from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing +from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable -class Fetcher: - def __init__( - self, - browser_engine: Optional[object] = None, - # Adaptor class parameters - response_encoding: str = "utf8", - huge_tree: bool = True, - keep_comments: Optional[bool] = False, - auto_match: Optional[bool] = False, - storage: Any = SQLiteStorageSystem, - storage_args: Optional[Dict] = None, - debug: Optional[bool] = True, - ): - if browser_engine is not None: - self.engine = check_if_engine_usable(browser_engine) - else: - self.engine = CamoufoxEngine() - # I won't validate Adaptor's class parameters here again, I will leave it to be validated later - self.__encoding = response_encoding - self.__huge_tree = huge_tree - self.__keep_comments = keep_comments - self.__auto_match = auto_match - self.__storage = storage - self.__storage_args = storage_args - self.__debug = debug +class Fetcher(BaseFetcher): + def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs) + return response_object - def __generate_adaptor(self, url, html_content): - """To make the code less repetitive and manage return result from one function""" - return Adaptor( - text=html_content, - url=url, - encoding=self.__encoding, - huge_tree=self.__huge_tree, - keep_comments=self.__keep_comments, - auto_match=self.__auto_match, - storage=self.__storage, - storage_args=self.__storage_args, - debug=self.__debug, + def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs) + return response_object + + def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs) + return response_object + + def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs) + return response_object + + +class StealthyFetcher(BaseFetcher): + def fetch( + self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False, + network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, + wait_selector_state: str = 'attached', + ) -> Response: + engine = CamoufoxEngine( + timeout=timeout, + headless=headless, + page_action=page_action, + block_images=block_images, + block_webrtc=block_webrtc, + network_idle=network_idle, + wait_selector=wait_selector, + wait_selector_state=wait_selector_state, + adaptor_arguments=self.adaptor_arguments, ) + return engine.fetch(url) - def fetch(self, url: str) -> Adaptor: - html_content = self.engine.fetch(url) - return self.__generate_adaptor(url, html_content) - def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor: - html_content = StaticEngine(follow_redirects, timeout).get(url, stealthy_headers, **kwargs) - return self.__generate_adaptor(url, html_content) +class PlayWrightFetcher(BaseFetcher): + def fetch( + self, + url: str, + headless: Union[bool, str] = True, + disable_resources: Optional[List] = None, + useragent: Optional[str] = None, + network_idle: Optional[bool] = False, + timeout: Optional[float] = 30000, + page_action: Callable = do_nothing, + wait_selector: Optional[str] = None, + wait_selector_state: Optional[str] = 'attached', + stealth: bool = False, + hide_canvas: bool = True, + disable_webgl: bool = False, + cdp_url: Optional[str] = None, + nstbrowser_mode: bool = False, + nstbrowser_config: Optional[Dict] = None, + ) -> Response: + engine = PlaywrightEngine( + timeout=timeout, + stealth=stealth, + cdp_url=cdp_url, + headless=headless, + useragent=useragent, + page_action=page_action, + hide_canvas=hide_canvas, + network_idle=network_idle, + wait_selector=wait_selector, + disable_webgl=disable_webgl, + nstbrowser_mode=nstbrowser_mode, + nstbrowser_config=nstbrowser_config, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, + adaptor_arguments=self.adaptor_arguments, + ) + return engine.fetch(url) - def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor: - html_content = StaticEngine(follow_redirects, timeout).post(url, stealthy_headers, **kwargs) - return self.__generate_adaptor(url, html_content) - def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor: - html_content = StaticEngine(follow_redirects, timeout).put(url, stealthy_headers, **kwargs) - return self.__generate_adaptor(url, html_content) - - def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor: - html_content = StaticEngine(follow_redirects, timeout).delete(url, stealthy_headers, **kwargs) - return self.__generate_adaptor(url, html_content) +class CustomFetcher(BaseFetcher): + def fetch(self, url: str, browser_engine, **kwargs) -> Response: + engine = check_if_engine_usable(browser_engine)(adaptor_arguments=self.adaptor_arguments, **kwargs) + return engine.fetch(url) diff --git a/scrapling/parser.py b/scrapling/parser.py index 771d08a..2dd4f66 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,12 +1,12 @@ import os from difflib import SequenceMatcher -from scrapling.translator import HTMLTranslator -from scrapling.mixins import SelectorsGeneration -from scrapling.custom_types import TextHandler, AttributesHandler -from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools -from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden -from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex +from scrapling.core.translator import HTMLTranslator +from scrapling.core.mixins import SelectorsGeneration +from scrapling.core.custom_types import TextHandler, AttributesHandler +from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools +from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden +from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex from lxml import etree, html from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors