freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Big structure changes (check commit description)

Browse Source 
- Moved most of the parser functions/files to the core package.
- Converted tools file to a package and made separate files for similar functions.
- Now all fetcher engines return a Response object
- Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing.
- I added a new custom fetcher so the user can create and use an engine.
- More...
This commit is contained in:
Karim shoair
2024-11-03 01:04:02 +02:00
parent 7f1c06e7bf
commit 145c03daff
21 changed files with 565 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/__init__.py
+3 -3
View File
@@ -1,11 +1,11 @@
# Declare top-level shortcuts
from scrapling.fetcher import Fetcher
from scrapling.fetcher import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
from scrapling.parser import Adaptor, Adaptors
from scrapling.custom_types import TextHandler, AttributesHandler
from scrapling.core.custom_types import TextHandler, AttributesHandler
__author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.2"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler', 'Fetcher']
__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher']
scrapling/core/__init__.py
View File
scrapling/_types.py → scrapling/core/_types.py
View File
scrapling/custom_types.py → scrapling/core/custom_types.py
+2 -2
View File
@@ -2,8 +2,8 @@ import re
from types import MappingProxyType
from collections.abc import Mapping
from scrapling.utils import _is_iterable, flatten
from scrapling._types import Dict, List, Union, Pattern
from scrapling.core.utils import _is_iterable, flatten
from scrapling.core._types import Dict, List, Union, Pattern
from orjson import loads, dumps
from w3lib.html import replace_entities as _replace_entities
scrapling/mixins.py → scrapling/core/mixins.py
View File
scrapling/storage_adaptors.py → scrapling/core/storage_adaptors.py
+2 -2
View File
@@ -5,8 +5,8 @@ import threading
from hashlib import sha256
from abc import ABC, abstractmethod
from scrapling._types import Dict, Optional, Union
from scrapling.utils import _StorageTools, cache
from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import _StorageTools, cache
from lxml import html
from tldextract import extract as tld
scrapling/translator.py → scrapling/core/translator.py
+2 -2
View File
@@ -9,8 +9,8 @@ which will be important in future releases but most importantly...
import re
from w3lib.html import HTML5_WHITESPACE
from scrapling.utils import cache
from scrapling._types import Any, Optional, Protocol, Self
from scrapling.core.utils import cache
from scrapling.core._types import Any, Optional, Protocol, Self
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
scrapling/utils.py → scrapling/core/utils.py
+1 -1
View File
@@ -4,7 +4,7 @@ from itertools import chain
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache
from scrapling._types import Dict, Iterable, Any
from scrapling.core._types import Dict, Iterable, Any
from lxml import html
scrapling/engines/__init__.py
+3 -2
View File
@@ -1,6 +1,7 @@
from .camo import CamoufoxEngine
from .static import StaticEngine
from .pw import PlaywrightEngine, DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .tools import check_if_engine_usable
from .pw import PlaywrightEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .toolbelt import check_if_engine_usable
__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
scrapling/engines/camo.py
+36 -15
View File
@@ -1,26 +1,28 @@
import logging
from scrapling._types import Union, Callable, Optional
from scrapling.core._types import Union, Callable, Optional, Dict
from .tools import check_type_validity, get_os_name, generate_convincing_referer
from scrapling.engines.toolbelt import (
Response,
do_nothing,
get_os_name,
check_type_validity,
generate_convincing_referer,
)
from camoufox.sync_api import Camoufox
def _do_nothing(page):
# Anything
return page
class CamoufoxEngine:
def __init__(
self, headless: Union[bool, str] = True,
block_images: Optional[bool] = True,
block_images: Optional[bool] = False,
block_webrtc: Optional[bool] = False,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = _do_nothing,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: str = 'attached',
adaptor_arguments: Dict = None
):
self.headless = headless
self.block_images = bool(block_images)
@@ -30,23 +32,24 @@ class CamoufoxEngine:
if callable(page_action):
self.page_action = page_action
else:
self.page_action = _do_nothing
self.page_action = do_nothing
logging.error('[Ignored] Argument "page_action" must be callable')
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
def fetch(self, url: str):
def fetch(self, url: str) -> Response:
with Camoufox(
headless=self.headless,
block_images=self.block_images,
block_images=self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
os=get_os_name(),
block_webrtc=self.block_webrtc,
) as browser:
page = browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.goto(url, referer=generate_convincing_referer(url))
res = page.goto(url, referer=generate_convincing_referer(url))
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
@@ -58,6 +61,24 @@ class CamoufoxEngine:
waiter = page.locator(self.wait_selector)
waiter.wait_for(state=self.wait_selector_state)
html = page.content()
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
response = Response(
url=res.url,
text=res.text(),
content=res.body(),
status=res.status,
reason=res.status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
request_headers=res.request.all_headers(),
adaptor_arguments=self.adaptor_arguments
)
page.close()
return html
return response
scrapling/engines/constants.py
+108
View File
@@ -0,0 +1,108 @@
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = [
'font',
'image',
'media',
'beacon',
'object',
'imageset',
'texttrack',
'websocket',
'csp_report',
'stylesheet',
]
DEFAULT_STEALTH_FLAGS = [
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--no-pings',
'--incognito',
'--test-type',
'--lang=en-US',
'--mute-audio',
'--no-first-run',
'--disable-sync',
'--hide-scrollbars',
'--disable-logging',
'--start-maximized', # For headless check bypass
'--enable-async-dns',
'--disable-breakpad',
'--disable-infobars',
'--accept-lang=en-US',
'--use-mock-keychain',
'--disable-translate',
'--disable-extensions',
'--disable-voice-input',
'--window-position=0,0',
'--disable-wake-on-wifi',
'--ignore-gpu-blocklist',
'--enable-tcp-fast-open',
'--enable-web-bluetooth',
'--disable-hang-monitor',
'--password-store=basic',
'--disable-cloud-import',
'--disable-default-apps',
'--disable-print-preview',
'--disable-dev-shm-usage',
'--disable-popup-blocking',
'--metrics-recording-only',
'--disable-crash-reporter',
'--disable-partial-raster',
'--disable-gesture-typing',
'--disable-checker-imaging',
'--disable-prompt-on-repost',
'--force-color-profile=srgb',
'--font-render-hinting=none',
'--no-default-browser-check',
'--aggressive-cache-discard',
'--disable-component-update',
'--disable-cookie-encryption',
'--disable-domain-reliability',
'--disable-threaded-animation',
'--disable-threaded-scrolling',
# '--disable-reading-from-canvas', # For Firefox
'--enable-simple-cache-backend',
'--disable-background-networking',
'--disable-session-crashed-bubble',
'--enable-surface-synchronization',
'--disable-image-animation-resync',
'--disable-renderer-backgrounding',
'--disable-ipc-flooding-protection',
'--prerender-from-omnibox=disabled',
'--safebrowsing-disable-auto-update',
'--disable-offer-upload-credit-cards',
'--disable-features=site-per-process',
'--disable-background-timer-throttling',
'--disable-new-content-rendering-timeout',
'--run-all-compositor-stages-before-draw',
'--disable-client-side-phishing-detection',
'--disable-backgrounding-occluded-windows',
'--disable-layer-tree-host-memory-pressure',
'--autoplay-policy=no-user-gesture-required',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-blink-features=AutomationControlled',
'--webrtc-ip-handling-policy=disable_non_proxied_udp',
'--disable-component-extensions-with-background-pages',
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
]
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
NSTBROWSER_DEFAULT_QUERY = {
"once": True,
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
}
scrapling/engines/pw.py
+37 -60
View File
@@ -1,43 +1,17 @@
import json
import logging
from scrapling._types import Union, Callable, Optional, List, Dict
from scrapling.core._types import Union, Callable, Optional, List, Dict
from .tools import check_type_validity, generate_headers, js_bypass_path, construct_websocket_url, generate_convincing_referer
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = ['beacon', 'csp_report', 'font', 'image', 'imageset', 'media', 'object', 'texttrack', 'stylesheet', 'websocket']
DEFAULT_STEALTH_FLAGS = [
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--incognito', '--accept-lang=en-US', '--lang=en-US', '--no-pings', '--mute-audio', '--no-first-run', '--no-default-browser-check', '--disable-cloud-import',
'--disable-gesture-typing', '--disable-offer-store-unmasked-wallet-cards', '--disable-offer-upload-credit-cards', '--disable-print-preview', '--disable-voice-input',
'--disable-wake-on-wifi', '--disable-cookie-encryption', '--ignore-gpu-blocklist', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open',
'--prerender-from-omnibox=disabled', '--enable-web-bluetooth', '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
'--aggressive-cache-discard', '--disable-ipc-flooding-protection', '--disable-blink-features=AutomationControlled', '--test-type',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--disable-breakpad', '--disable-component-update', '--disable-domain-reliability', '--disable-sync', '--disable-client-side-phishing-detection',
'--disable-hang-monitor', '--disable-popup-blocking', '--disable-prompt-on-repost', '--metrics-recording-only', '--safebrowsing-disable-auto-update', '--password-store=basic',
'--autoplay-policy=no-user-gesture-required', '--use-mock-keychain', '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--webrtc-ip-handling-policy=disable_non_proxied_udp', '--disable-session-crashed-bubble', '--disable-crash-reporter', '--disable-dev-shm-usage', '--force-color-profile=srgb',
'--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-infobars',
'--hide-scrollbars', '--disable-renderer-backgrounding', '--font-render-hinting=none', '--disable-logging', '--enable-surface-synchronization',
'--run-all-compositor-stages-before-draw', '--disable-threaded-animation', '--disable-threaded-scrolling', '--disable-checker-imaging',
'--disable-new-content-rendering-timeout', '--disable-image-animation-resync', '--disable-partial-raster',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-layer-tree-host-memory-pressure',
'--window-position=0,0',
'--disable-features=site-per-process',
'--disable-default-apps',
'--disable-component-extensions-with-background-pages',
'--disable-extensions',
# "--disable-reading-from-canvas", # For Firefox
'--start-maximized' # For headless check bypass
]
def _do_nothing(page):
# Anything
return page
from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
from scrapling.engines.toolbelt import (
Response,
do_nothing,
js_bypass_path,
generate_headers,
check_type_validity,
construct_websocket_url,
generate_convincing_referer,
)
class PlaywrightEngine:
@@ -47,7 +21,7 @@ class PlaywrightEngine:
useragent: Optional[str] = None,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = _do_nothing,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: Optional[str] = 'attached',
stealth: bool = False,
@@ -56,6 +30,7 @@ class PlaywrightEngine:
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
adaptor_arguments: Dict = None
):
self.headless = headless
self.disable_resources = disable_resources
@@ -69,13 +44,14 @@ class PlaywrightEngine:
if callable(page_action):
self.page_action = page_action
else:
self.page_action = _do_nothing
self.page_action = do_nothing
logging.error('[Ignored] Argument "page_action" must be callable')
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.nstbrowser_mode = bool(nstbrowser_mode)
self.nstbrowser_config = nstbrowser_config
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
def _cdp_url_logic(self, flags: Optional[dict] = None):
cdp_url = self.cdp_url
@@ -83,23 +59,7 @@ class PlaywrightEngine:
if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
config = self.nstbrowser_config
else:
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
query = {
"once": True,
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
}
query = NSTBROWSER_DEFAULT_QUERY.copy()
if flags:
query.update({
"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary
@@ -113,7 +73,7 @@ class PlaywrightEngine:
return cdp_url
def fetch(self, url):
def fetch(self, url) -> Response:
if not self.stealth:
from playwright.sync_api import sync_playwright
else:
@@ -192,7 +152,7 @@ class PlaywrightEngine:
page.add_init_script(path=js_bypass_path('screen_props.js'))
page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
res = page.goto(url, referer=generate_convincing_referer(url) if self.stealth else None)
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
@@ -204,6 +164,23 @@ class PlaywrightEngine:
waiter = page.locator(self.wait_selector)
waiter.wait_for(state=self.wait_selector_state)
html = page.content()
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
response = Response(
url=res.url,
text=res.text(),
content=res.body(),
status=res.status,
reason=res.status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
request_headers=res.request.all_headers(),
adaptor_arguments=self.adaptor_arguments
)
page.close()
return html
return response
scrapling/engines/static.py
+23 -6
View File
@@ -1,9 +1,10 @@
import logging
from scrapling._types import Union, Optional, Dict
from .tools import generate_convincing_referer, generate_headers
from scrapling.core._types import Union, Optional, Dict
from .toolbelt import Response, generate_convincing_referer, generate_headers
import httpx
from httpx._models import Response as httpxResponse
class StaticEngine:
@@ -11,10 +12,12 @@ class StaticEngine:
self,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None,
adaptor_arguments: Dict = None
):
self.timeout = timeout
self.follow_redirects = bool(follow_redirects)
self._extra_headers = generate_headers(browser_mode=False)
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
@staticmethod
def _headers_job(headers, url, stealth):
@@ -32,22 +35,36 @@ class StaticEngine:
return headers
def _prepare_response(self, response: httpxResponse):
return Response(
url=str(response.url),
text=response.text,
content=response.content,
status=response.status_code,
reason=response.reason_phrase,
encoding=response.encoding or 'utf-8',
cookies=dict(response.cookies),
headers=dict(response.headers),
request_headers=response.request.headers,
adaptor_arguments=self.adaptor_arguments
)
def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict):
headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return request.text
return self._prepare_response(request)
scrapling/engines/toolbelt/__init__.py
+18
View File
@@ -0,0 +1,18 @@
from .fingerprints import (
get_os_name,
generate_headers,
generate_convincing_referer,
generate_suitable_fingerprint,
)
from .custom import (
Response,
do_nothing,
BaseFetcher,
get_variable_name,
check_type_validity,
check_if_engine_usable,
)
from .navigation import (
js_bypass_path,
construct_websocket_url,
)
scrapling/engines/toolbelt/custom.py
+136
View File
@@ -0,0 +1,136 @@
"""
Functions related to custom types or type checking
"""
import inspect
import logging
from dataclasses import dataclass, field
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.core._types import Any, List, Type, Union, Optional, Dict
@dataclass(frozen=True)
class Response:
url: str
text: str
content: bytes
status: int
reason: str
encoding: str = 'utf-8' # default encoding
cookies: Dict = field(default_factory=dict)
headers: Dict = field(default_factory=dict)
request_headers: Dict = field(default_factory=dict)
adaptor_arguments: Dict = field(default_factory=dict)
@property
def adaptor(self):
if self.text:
return Adaptor(text=self.text, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
elif self.content:
return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
return None
def __repr__(self):
return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
class BaseFetcher:
def __init__(
self,
# Adaptor class parameters
huge_tree: bool = True,
keep_comments: Optional[bool] = False,
auto_match: Optional[bool] = False,
storage: Any = SQLiteStorageSystem,
storage_args: Optional[Dict] = None,
debug: Optional[bool] = True,
):
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
self.adaptor_arguments = dict(
huge_tree=huge_tree,
keep_comments=keep_comments,
auto_match=auto_match,
storage=storage,
storage_args=storage_args,
debug=debug,
)
def check_if_engine_usable(engine):
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
:return: The name of the variable if found, None otherwise
"""
for scope in [globals(), locals()]:
for name, value in scope.items():
if value is var:
return name
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
return variable
# Pew Pew
def do_nothing(page):
# Just works as a filler for `page_action` argument in browser engines
return page
scrapling/engines/toolbelt/fingerprints.py
+65
View File
@@ -0,0 +1,65 @@
"""
Functions related to generating headers and fingerprints generally
"""
import platform
from tldextract import extract
from browserforge.fingerprints import FingerprintGenerator
from browserforge.headers import HeaderGenerator, Browser
def generate_convincing_referer(url):
"""
Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
:param url: The URL you are about to fetch.
:return:
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
def get_os_name():
# Get the OS name in the same format needed for browserforge
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
# For the future? because why not
'iOS': 'ios',
}.get(os_name)
def generate_suitable_fingerprint():
# This would be for Browserforge playwright injector
os_name = get_os_name()
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
def generate_headers(browser_mode=False):
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
scrapling/engines/toolbelt/navigation.py
+43
View File
@@ -0,0 +1,43 @@
"""
Functions related to files and URLs
"""
import os
from urllib.parse import urlparse, urlencode
def construct_websocket_url(base_url, query_params):
# Validate the base URL structure
try:
parsed = urlparse(base_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
raise ValueError("URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname")
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
# Add query parameters
if query_params:
query_string = urlencode(query_params)
return f"{validated_base}?{query_string}"
return validated_base
except Exception as e:
raise ValueError(f"Invalid WebSocket URL: {str(e)}")
def js_bypass_path(filename):
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
scrapling/engines/tools.py
-174
View File
@@ -1,174 +0,0 @@
import os
import logging
import inspect
import platform
from scrapling._types import Any, List, Type, Union, Optional
from urllib.parse import urlparse, urlencode
from tldextract import extract
from browserforge.fingerprints import FingerprintGenerator
from browserforge.headers import HeaderGenerator, Browser
def generate_convincing_referer(url):
"""
Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
:param url: The URL you are about to fetch.
:return:
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
def check_if_engine_usable(engine):
if isinstance(engine, type):
raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
else:
raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
else:
raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
def construct_websocket_url(base_url, query_params):
# Validate the base URL structure
try:
parsed = urlparse(base_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
raise ValueError("URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname")
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
# Add query parameters
if query_params:
query_string = urlencode(query_params)
return f"{validated_base}?{query_string}"
return validated_base
except Exception as e:
raise ValueError(f"Invalid WebSocket URL: {str(e)}")
def js_bypass_path(filename):
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
def get_os_name():
# Get the OS name in the same format needed for browserforge
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
# For the future? because why not
'iOS': 'ios',
}.get(os_name)
def generate_suitable_fingerprint():
# This would be for Browserforge playwright injector
os_name = get_os_name()
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
def generate_headers(browser_mode=False):
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=os_name, # None is ignored
device='desktop'
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
:return: The name of the variable if found, None otherwise
"""
for scope in [globals(), locals()]:
for name, value in scope.items():
if value is var:
return name
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
logging.error(f'[Ignored] {error_msg}')
return default_value
return variable
scrapling/fetcher.py
+78 -56
View File
@@ -1,65 +1,87 @@
from scrapling._types import Any, Dict, Optional, Union
from scrapling.core._types import Dict, Optional, Union, Callable, List
from scrapling.engines import CamoufoxEngine, StaticEngine, check_if_engine_usable
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing
from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable
class Fetcher:
def __init__(
self,
browser_engine: Optional[object] = None,
# Adaptor class parameters
response_encoding: str = "utf8",
huge_tree: bool = True,
keep_comments: Optional[bool] = False,
auto_match: Optional[bool] = False,
storage: Any = SQLiteStorageSystem,
storage_args: Optional[Dict] = None,
debug: Optional[bool] = True,
):
if browser_engine is not None:
self.engine = check_if_engine_usable(browser_engine)
else:
self.engine = CamoufoxEngine()
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
self.__encoding = response_encoding
self.__huge_tree = huge_tree
self.__keep_comments = keep_comments
self.__auto_match = auto_match
self.__storage = storage
self.__storage_args = storage_args
self.__debug = debug
class Fetcher(BaseFetcher):
def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs)
return response_object
def __generate_adaptor(self, url, html_content):
"""To make the code less repetitive and manage return result from one function"""
return Adaptor(
text=html_content,
url=url,
encoding=self.__encoding,
huge_tree=self.__huge_tree,
keep_comments=self.__keep_comments,
auto_match=self.__auto_match,
storage=self.__storage,
storage_args=self.__storage_args,
debug=self.__debug,
def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs)
return response_object
def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
return response_object
def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs)
return response_object
class StealthyFetcher(BaseFetcher):
def fetch(
self, url: str, headless: Union[bool, str] = True, block_images: Optional[bool] = False, block_webrtc: Optional[bool] = False,
network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None,
wait_selector_state: str = 'attached',
) -> Response:
engine = CamoufoxEngine(
timeout=timeout,
headless=headless,
page_action=page_action,
block_images=block_images,
block_webrtc=block_webrtc,
network_idle=network_idle,
wait_selector=wait_selector,
wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
def fetch(self, url: str) -> Adaptor:
html_content = self.engine.fetch(url)
return self.__generate_adaptor(url, html_content)
def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).get(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
class PlayWrightFetcher(BaseFetcher):
def fetch(
self,
url: str,
headless: Union[bool, str] = True,
disable_resources: Optional[List] = None,
useragent: Optional[str] = None,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
page_action: Callable = do_nothing,
wait_selector: Optional[str] = None,
wait_selector_state: Optional[str] = 'attached',
stealth: bool = False,
hide_canvas: bool = True,
disable_webgl: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
) -> Response:
engine = PlaywrightEngine(
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
headless=headless,
useragent=useragent,
page_action=page_action,
hide_canvas=hide_canvas,
network_idle=network_idle,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
nstbrowser_mode=nstbrowser_mode,
nstbrowser_config=nstbrowser_config,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).post(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).put(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Adaptor:
html_content = StaticEngine(follow_redirects, timeout).delete(url, stealthy_headers, **kwargs)
return self.__generate_adaptor(url, html_content)
class CustomFetcher(BaseFetcher):
def fetch(self, url: str, browser_engine, **kwargs) -> Response:
engine = check_if_engine_usable(browser_engine)(adaptor_arguments=self.adaptor_arguments, **kwargs)
return engine.fetch(url)
scrapling/parser.py
+6 -6
View File
@@ -1,12 +1,12 @@
import os
from difflib import SequenceMatcher
from scrapling.translator import HTMLTranslator
from scrapling.mixins import SelectorsGeneration
from scrapling.custom_types import TextHandler, AttributesHandler
from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
from scrapling.core.translator import HTMLTranslator
from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.custom_types import TextHandler, AttributesHandler
from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
from lxml import etree, html
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors