freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Big structure changes (check commit description)

Browse Source 
- Moved most of the parser functions/files to the core package.
- Converted tools file to a package and made separate files for similar functions.
- Now all fetcher engines return a Response object
- Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing.
- I added a new custom fetcher so the user can create and use an engine.
- More...
This commit is contained in:
Karim shoair
2024-11-03 01:04:02 +02:00
parent 7f1c06e7bf
commit 145c03daff
21 changed files with 565 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -477,6 +477,8 @@ This project includes code adapted from:
- [brotector](https://github.com/kaliiiiiiiiii/brotector)
- [fakebrowser](https://github.com/kkoooqq/fakebrowser)
- [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches)
- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright)
- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
## Known Issues
- In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone.
scrapling/__init__.py
+3 -3
View File
@@ -1,11 +1,11 @@
# Declare top-level shortcuts
from scrapling.fetcher import Fetcher
from scrapling.fetcher import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
from scrapling.parser import Adaptor, Adaptors
from scrapling.custom_types import TextHandler, AttributesHandler
from scrapling.core.custom_types import TextHandler, AttributesHandler
__author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.2"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler', 'Fetcher']
__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher']
scrapling/core/__init__.py
View File
scrapling/_types.py → scrapling/core/_types.py
View File
scrapling/custom_types.py → scrapling/core/custom_types.py
+2 -2
View File
@@ -2,8 +2,8 @@ import re
from types import MappingProxyType
from collections.abc import Mapping
from scrapling.utils import _is_iterable, flatten
from scrapling._types import Dict, List, Union, Pattern
from scrapling.core.utils import _is_iterable, flatten
from scrapling.core._types import Dict, List, Union, Pattern
from orjson import loads, dumps
from w3lib.html import replace_entities as _replace_entities
scrapling/mixins.py → scrapling/core/mixins.py
View File
scrapling/storage_adaptors.py → scrapling/core/storage_adaptors.py
+2 -2
View File
@@ -5,8 +5,8 @@ import threading
from hashlib import sha256
from abc import ABC, abstractmethod
from scrapling._types import Dict, Optional, Union
from scrapling.utils import _StorageTools, cache
from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import _StorageTools, cache
from lxml import html
from tldextract import extract as tld
scrapling/translator.py → scrapling/core/translator.py
+2 -2
View File
@@ -9,8 +9,8 @@ which will be important in future releases but most importantly...
import re
from w3lib.html import HTML5_WHITESPACE
from scrapling.utils import cache
from scrapling._types import Any, Optional, Protocol, Self
from scrapling.core.utils import cache
from scrapling.core._types import Any, Optional, Protocol, Self
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
scrapling/utils.py → scrapling/core/utils.py
+1 -1
View File
@@ -4,7 +4,7 @@ from itertools import chain
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache
from scrapling._types import Dict, Iterable, Any
from scrapling.core._types import Dict, Iterable, Any
from lxml import html
scrapling/engines/__init__.py
+3 -2
View File
@@ -1,6 +1,7 @@
from .camo import CamoufoxEngine
from .static import StaticEngine
from .pw import PlaywrightEngine, DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .tools import check_if_engine_usable
from .pw import PlaywrightEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .toolbelt import check_if_engine_usable
__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
scrapling/engines/camo.py
+36 -15
View File
@@ -1,26 +1,28 @@
import logging
from scrapling._types import Union, Callable, Optional
from scrapling.core._types import Union, Callable, Optional, Dict
from .tools import check_type_validity, get_os_name, generate_convincing_referer
from scrapling.engines.toolbelt import (
Response,
do_nothing,
get_os_name,
check_type_validity,
generate_convincing_referer,
)
from camoufox.sync_api import Camoufox
def _do_nothing(page):
# Anything
return page
class CamoufoxEngine:
def __init__(
self, headless: Union[bool, str] = True,
block_images: Optional[bool] = True,
block_images: Optional[bool] = False,
block_webrtc: Optional[bool] = False,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = _do_nothing,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: str = 'attached',
adaptor_arguments: Dict = None
):
self.headless = headless
self.block_images = bool(block_images)
@@ -30,23 +32,24 @@ class CamoufoxEngine:
if callable(page_action):
self.page_action = page_action
else:
self.page_action = _do_nothing
self.page_action = do_nothing
logging.error('[Ignored] Argument "page_action" must be callable')
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
def fetch(self, url: str):
def fetch(self, url: str) -> Response:
with Camoufox(
headless=self.headless,
block_images=self.block_images,
block_images=self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
os=get_os_name(),
block_webrtc=self.block_webrtc,
) as browser:
page = browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.goto(url, referer=generate_convincing_referer(url))
res = page.goto(url, referer=generate_convincing_referer(url))
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
@@ -58,6 +61,24 @@ class CamoufoxEngine:
waiter = page.locator(self.wait_selector)
waiter.wait_for(state=self.wait_selector_state)
html = page.content()
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
response = Response(
url=res.url,
text=res.text(),
content=res.body(),
status=res.status,
reason=res.status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
request_headers=res.request.all_headers(),
adaptor_arguments=self.adaptor_arguments
)
page.close()
return html
return response
scrapling/engines/constants.py
+108
View File
@@ -0,0 +1,108 @@
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = [
'font',
'image',
'media',
'beacon',
'object',
'imageset',
'texttrack',
'websocket',
'csp_report',
'stylesheet',
]
DEFAULT_STEALTH_FLAGS = [
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--no-pings',
'--incognito',
'--test-type',
'--lang=en-US',
'--mute-audio',
'--no-first-run',
'--disable-sync',
'--hide-scrollbars',
'--disable-logging',
'--start-maximized', # For headless check bypass
'--enable-async-dns',
'--disable-breakpad',
'--disable-infobars',
'--accept-lang=en-US',
'--use-mock-keychain',
'--disable-translate',
'--disable-extensions',
'--disable-voice-input',
'--window-position=0,0',
'--disable-wake-on-wifi',
'--ignore-gpu-blocklist',
'--enable-tcp-fast-open',
'--enable-web-bluetooth',
'--disable-hang-monitor',
'--password-store=basic',
'--disable-cloud-import',
'--disable-default-apps',
'--disable-print-preview',
'--disable-dev-shm-usage',
'--disable-popup-blocking',
'--metrics-recording-only',
'--disable-crash-reporter',
'--disable-partial-raster',
'--disable-gesture-typing',
'--disable-checker-imaging',
'--disable-prompt-on-repost',
'--force-color-profile=srgb',
'--font-render-hinting=none',
'--no-default-browser-check',
'--aggressive-cache-discard',
'--disable-component-update',
'--disable-cookie-encryption',
'--disable-domain-reliability',
'--disable-threaded-animation',
'--disable-threaded-scrolling',
# '--disable-reading-from-canvas', # For Firefox
'--enable-simple-cache-backend',
'--disable-background-networking',
'--disable-session-crashed-bubble',
'--enable-surface-synchronization',
'--disable-image-animation-resync',
'--disable-renderer-backgrounding',
'--disable-ipc-flooding-protection',
'--prerender-from-omnibox=disabled',
'--safebrowsing-disable-auto-update',
'--disable-offer-upload-credit-cards',
'--disable-features=site-per-process',
'--disable-background-timer-throttling',
'--disable-new-content-rendering-timeout',
'--run-all-compositor-stages-before-draw',
'--disable-client-side-phishing-detection',
'--disable-backgrounding-occluded-windows',
'--disable-layer-tree-host-memory-pressure',
'--autoplay-policy=no-user-gesture-required',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-blink-features=AutomationControlled',
'--webrtc-ip-handling-policy=disable_non_proxied_udp',
'--disable-component-extensions-with-background-pages',
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
]
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
NSTBROWSER_DEFAULT_QUERY = {
"once": True,
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
}
scrapling/engines/pw.py
+37 -60
View File
@@ -1,43 +1,17 @@
import json
import logging
from scrapling._types import Union, Callable, Optional, List, Dict
from scrapling.core._types import Union, Callable, Optional, List, Dict
from .tools import check_type_validity, generate_headers, js_bypass_path, construct_websocket_url, generate_convincing_referer
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = ['beacon', 'csp_report', 'font', 'image', 'imageset', 'media', 'object', 'texttrack', 'stylesheet', 'websocket']
DEFAULT_STEALTH_FLAGS = [
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--incognito', '--accept-lang=en-US', '--lang=en-US', '--no-pings', '--mute-audio', '--no-first-run', '--no-default-browser-check', '--disable-cloud-import',
'--disable-gesture-typing', '--disable-offer-store-unmasked-wallet-cards', '--disable-offer-upload-credit-cards', '--disable-print-preview', '--disable-voice-input',
'--disable-wake-on-wifi', '--disable-cookie-encryption', '--ignore-gpu-blocklist', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open',
'--prerender-from-omnibox=disabled', '--enable-web-bluetooth', '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
'--aggressive-cache-discard', '--disable-ipc-flooding-protection', '--disable-blink-features=AutomationControlled', '--test-type',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--disable-breakpad', '--disable-component-update', '--disable-domain-reliability', '--disable-sync', '--disable-client-side-phishing-detection',
'--disable-hang-monitor', '--disable-popup-blocking', '--disable-prompt-on-repost', '--metrics-recording-only', '--safebrowsing-disable-auto-update', '--password-store=basic',
'--autoplay-policy=no-user-gesture-required', '--use-mock-keychain', '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--webrtc-ip-handling-policy=disable_non_proxied_udp', '--disable-session-crashed-bubble', '--disable-crash-reporter', '--disable-dev-shm-usage', '--force-color-profile=srgb',
'--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-infobars',
'--hide-scrollbars', '--disable-renderer-backgrounding', '--font-render-hinting=none', '--disable-logging', '--enable-surface-synchronization',
'--run-all-compositor-stages-before-draw', '--disable-threaded-animation', '--disable-threaded-scrolling', '--disable-checker-imaging',
'--disable-new-content-rendering-timeout', '--disable-image-animation-resync', '--disable-partial-raster',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-layer-tree-host-memory-pressure',
'--window-position=0,0',
'--disable-features=site-per-process',
'--disable-default-apps',
'--disable-component-extensions-with-background-pages',
'--disable-extensions',
# "--disable-reading-from-canvas", # For Firefox
'--start-maximized' # For headless check bypass
]
def _do_nothing(page):
# Anything
return page
from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
from scrapling.engines.toolbelt import (
Response,
do_nothing,
js_bypass_path,
generate_headers,
check_type_validity,
construct_websocket_url,
generate_convincing_referer,
)
class PlaywrightEngine:
@@ -47,7 +21,7 @@ class PlaywrightEngine:
useragent: Optional[str] = None,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = _do_nothing,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: Optional[str] = 'attached',
stealth: bool = False,
@@ -56,6 +30,7 @@ class PlaywrightEngine:
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
adaptor_arguments: Dict = None
):
self.headless = headless
self.disable_resources = disable_resources
@@ -69,13 +44,14 @@ class PlaywrightEngine:
if callable(page_action):
self.page_action = page_action
else:
self.page_action = _do_nothing
self.page_action = do_nothing
logging.error('[Ignored] Argument "page_action" must be callable')
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.nstbrowser_mode = bool(nstbrowser_mode)
self.nstbrowser_config = nstbrowser_config
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
def _cdp_url_logic(self, flags: Optional[dict] = None):
cdp_url = self.cdp_url
@@ -83,23 +59,7 @@ class PlaywrightEngine:
if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
config = self.nstbrowser_config
else:
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
query = {
"once": True,
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
}
query = NSTBROWSER_DEFAULT_QUERY.copy()
if flags:
query.update({
"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary
@@ -113,7 +73,7 @@ class PlaywrightEngine:
return cdp_url
def fetch(self, url):
def fetch(self, url) -> Response:
if not self.stealth:
from playwright.sync_api import sync_playwright
else:
@@ -192,7 +152,7 @@ class PlaywrightEngine:
page.add_init_script(path=js_bypass_path('screen_props.js'))
page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
res = page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
@@ -204,6 +164,23 @@ class PlaywrightEngine:
waiter = page.locator(self.wait_selector)
waiter.wait_for(state=self.wait_selector_state)
html = page.content()
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
response = Response(
url=res.url,
text=res.text(),
content=res.body(),
status=res.status,
reason=res.status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
request_headers=res.request.all_headers(),
adaptor_arguments=self.adaptor_arguments
)
page.close()
return html
return response
scrapling/engines/static.py
+23 -6
View File
@@ -1,9 +1,10 @@
import logging
from scrapling._types import Union, Optional, Dict
from .tools import generate_convincing_referer, generate_headers
from scrapling.core._types import Union, Optional, Dict
from .toolbelt import Response, generate_convincing_referer, generate_headers
import httpx
from httpx._models import Response as httpxResponse
class StaticEngine:
@@ -11,10 +12,12 @@ class StaticEngine:
self,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None,
adaptor_arguments: Dict = None
):
self.timeout = timeout
self.follow_redirects = bool(follow_redirects)
self._extra_headers = generate_headers(browser_mode=False)
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
@staticmethod
def _headers_job(headers, url, stealth):
@@ -32,22 +35,36 @@ class StaticEngine:
return headers
def _prepare_response(self, response: httpxResponse):
return Response(
url=str(response.url),
text=response.text,
content=response.content,
status=response.status_code,
reason=response.reason_phrase,
encoding=response.encoding or 'utf-8',
cookies=dict(response.cookies),
headers=dict(response.headers),
request_headers=response.request.headers,
adaptor_arguments=self.adaptor_arguments
)
def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
scrapling/engines/toolbelt/__init__.py
+18
View File
@@ -0,0 +1,18 @@
from .fingerprints import (
get_os_name,
generate_headers,
generate_convincing_referer,
generate_suitable_fingerprint,
)
from .custom import (
Response,
do_nothing,
BaseFetcher,
get_variable_name,
check_type_validity,
check_if_engine_usable,
)
from .navigation import (
js_bypass_path,
construct_websocket_url,
)
scrapling/engines/toolbelt/custom.py
+136
View File
@@ -0,0 +1,136 @@
"""
Functions related to custom types or type checking
"""
import inspect
import logging
from dataclasses import dataclass, field
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.core._types import Any, List, Type, Union, Optional, Dict
@dataclass(frozen=True)
class Response:
url: str
text: str
content: bytes
status: int
reason: str
encoding: str = 'utf-8' # default encoding
cookies: Dict = field(default_factory=dict)
headers: Dict = field(default_factory=dict)
request_headers: Dict = field(default_factory=dict)
adaptor_arguments: Dict = field(default_factory=dict)
@property
def adaptor(self):
if self.text:
return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
elif self.content:
return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
return None
def __repr__(self):
return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
class BaseFetcher:
def __init__(
self,
# Adaptor class parameters
huge_tree: bool = True,
keep_comments: Optional[bool] = False,
auto_match: Optional[bool] = False,
storage: Any = SQLiteStorageSystem,
storage_args: Optional[Dict] = None,
debug: Optional[bool] = True,
):
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
self.adaptor_arguments = dict(
huge_tree=huge_tree,
keep_comments=keep_comments,
auto_match=auto_match,
storage=storage,
storage_args=storage_args,
debug=debug,
)
def check_if_engine_usable(engine):
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
:return: The name of the variable if found, None otherwise
"""
for scope in [globals(), locals()]:
for name, value in scope.items():
if value is var:
return name
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
return variable
# Pew Pew
def do_nothing(page):
# Just works as a filler for `page_action` argument in browser engines
return page
scrapling/engines/toolbelt/fingerprints.py
+65
View File
@@ -0,0 +1,65 @@
"""
Functions related to generating headers and fingerprints generally
"""
import platform
from tldextract import extract
from browserforge.fingerprints import FingerprintGenerator
from browserforge.headers import HeaderGenerator, Browser
def generate_convincing_referer(url):
"""
Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
:param url: The URL you are about to fetch.
:return:
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
def get_os_name():
# Get the OS name in the same format needed for browserforge
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
# For the future? because why not
'iOS': 'ios',
}.get(os_name)
def generate_suitable_fingerprint():
# This would be for Browserforge playwright injector
os_name = get_os_name()
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
def generate_headers(browser_mode=False):
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
scrapling/engines/toolbelt/navigation.py
+43
View File
@@ -0,0 +1,43 @@
"""
Functions related to files and URLs
"""
import os
from urllib.parse import urlparse, urlencode
def construct_websocket_url(base_url, query_params):
# Validate the base URL structure
try:
parsed = urlparse(base_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
raise ValueError("URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname")
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
# Add query parameters
if query_params:
query_string = urlencode(query_params)
return f"{validated_base}?{query_string}"
return validated_base
except Exception as e:
raise ValueError(f"Invalid WebSocket URL: {str(e)}")
def js_bypass_path(filename):
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
scrapling/engines/tools.py
-174
View File
@@ -1,174 +0,0 @@
import os
import logging
import inspect
import platform
from scrapling._types import Any, List, Type, Union, Optional
from urllib.parse import urlparse, urlencode
from tldextract import extract
from browserforge.fingerprints import FingerprintGenerator
from browserforge.headers import HeaderGenerator, Browser
def generate_convincing_referer(url):
"""
Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
:param url: The URL you are about to fetch.
:return:
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
def check_if_engine_usable(engine):
if isinstance(engine, type):
raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
else:
raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
else:
raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
def construct_websocket_url(base_url, query_params):
# Validate the base URL structure
try:
parsed = urlparse(base_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
raise ValueError("URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname")
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
# Add query parameters
if query_params:
query_string = urlencode(query_params)
return f"{validated_base}?{query_string}"
return validated_base
except Exception as e:
raise ValueError(f"Invalid WebSocket URL: {str(e)}")
def js_bypass_path(filename):
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
def get_os_name():
# Get the OS name in the same format needed for browserforge
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
# For the future? because why not
'iOS': 'ios',
}.get(os_name)
def generate_suitable_fingerprint():
# This would be for Browserforge playwright injector
os_name = get_os_name()
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
def generate_headers(browser_mode=False):
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
:return: The name of the variable if found, None otherwise
"""
for scope in [globals(), locals()]:
for name, value in scope.items():
if value is var:
return name
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
return variable
scrapling/fetcher.py
+78 -56
View File
@@ -1,65 +1,87 @@
from scrapling._types import Any, Dict, Optional, Union
from scrapling.core._types import Dict, Optional, Union, Callable, List
from scrapling.engines import CamoufoxEngine, StaticEngine, check_if_engine_usable
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing
from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable
class Fetcher:
def __init__(
self,
browser_engine: Optional[object] = None,
# Adaptor class parameters
response_encoding: str = "utf8",
huge_tree: bool = True,
keep_comments: Optional[bool] = False,
auto_match: Optional[bool] = False,
storage: Any = SQLiteStorageSystem,
storage_args: Optional[Dict] = None,
debug: Optional[bool] = True,
):
if browser_engine is not None:
self.engine = check_if_engine_usable(browser_engine)
else:
self.engine = CamoufoxEngine()
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
self.__encoding = response_encoding
self.__huge_tree = huge_tree
self.__keep_comments = keep_comments
self.__auto_match = auto_match
self.__storage = storage
self.__storage_args = storage_args
self.__debug = debug
class Fetcher(BaseFetcher):
def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs)
return response_object
def __generate_adaptor(self, url, html_content):
"""To make the code less repetitive and manage return result from one function"""
return Adaptor(
text=html_content,
url=url,
encoding=self.__encoding,
huge_tree=self.__huge_tree,
keep_comments=self.__keep_comments,
auto_match=self.__auto_match,
storage=self.__storage,
storage_args=self.__storage_args,
debug=self.__debug,
def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs)
return response_object
def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
return response_object
def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs)
return response_object
class StealthyFetcher(BaseFetcher):
def fetch(
self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False,
network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
wait_selector_state: str = 'attached',
) -> Response:
engine = CamoufoxEngine(
timeout=timeout,
headless=headless,
page_action=page_action,
block_images=block_images,
block_webrtc=block_webrtc,
network_idle=network_idle,
wait_selector=wait_selector,
wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
def fetch(self, url: str) -> Adaptor:
html_content = self.engine.fetch(url)
return self.__generate_adaptor(url, html_content)
def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).get(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
class PlayWrightFetcher(BaseFetcher):
def fetch(
self,
url: str,
headless: Union[bool, str] = True,
disable_resources: Optional[List] = None,
useragent: Optional[str] = None,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: Optional[str] = 'attached',
stealth: bool = False,
hide_canvas: bool = True,
disable_webgl: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
) -> Response:
engine = PlaywrightEngine(
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
headless=headless,
useragent=useragent,
page_action=page_action,
hide_canvas=hide_canvas,
network_idle=network_idle,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
nstbrowser_mode=nstbrowser_mode,
nstbrowser_config=nstbrowser_config,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).post(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).put(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).delete(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
class CustomFetcher(BaseFetcher):
def fetch(self, url: str, browser_engine, **kwargs) -> Response:
engine = check_if_engine_usable(browser_engine)(adaptor_arguments=self.adaptor_arguments, **kwargs)
return engine.fetch(url)
scrapling/parser.py
+6 -6
View File
@@ -1,12 +1,12 @@
import os
from difflib import SequenceMatcher
from scrapling.translator import HTMLTranslator
from scrapling.mixins import SelectorsGeneration
from scrapling.custom_types import TextHandler, AttributesHandler
from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
from scrapling.core.translator import HTMLTranslator
from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.custom_types import TextHandler, AttributesHandler
from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
from lxml import etree, html
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors