freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/__init__.py
+1 -1
View File
@@ -4,4 +4,4 @@ from .pw import PlaywrightEngine
from .static import StaticEngine
from .toolbelt import check_if_engine_usable
__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
__all__ = ["CamoufoxEngine", "PlaywrightEngine"]
scrapling/engines/camo.py
+125 -59
View File
@@ -2,27 +2,52 @@ from camoufox import DefaultAddons
from camoufox.async_api import AsyncCamoufox
from camoufox.sync_api import Camoufox
from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
SelectorWaitStates, Union)
from scrapling.core._types import (
Callable,
Dict,
List,
Literal,
Optional,
SelectorWaitStates,
Union,
)
from scrapling.core.utils import log
from scrapling.engines.toolbelt import (Response, StatusText,
async_intercept_route,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
get_os_name, intercept_route)
from scrapling.engines.toolbelt import (
Response,
StatusText,
async_intercept_route,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
get_os_name,
intercept_route,
)
class CamoufoxEngine:
def __init__(
self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Dict = None,
additional_arguments: Dict = None
self,
headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
wait: Optional[int] = 0,
timeout: Optional[float] = 30000,
page_action: Callable = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Dict = None,
additional_arguments: Dict = None,
):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -97,7 +122,7 @@ class CamoufoxEngine:
"block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
"os": None if self.os_randomize else get_os_name(),
**self.additional_arguments
**self.additional_arguments,
}
def _process_response_history(self, first_response):
@@ -109,19 +134,30 @@ class CamoufoxEngine:
while current_request:
try:
current_response = current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments
))
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text="",
body=b"",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=current_response.all_headers()
if current_response
else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
@@ -141,19 +177,30 @@ class CamoufoxEngine:
while current_request:
try:
current_response = await current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments
))
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text="",
body=b"",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=await current_response.all_headers()
if current_response
else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
@@ -175,7 +222,10 @@ class CamoufoxEngine:
def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
with Camoufox(**self._get_camoufox_options()) as browser:
@@ -195,7 +245,7 @@ class CamoufoxEngine:
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
@@ -211,7 +261,7 @@ class CamoufoxEngine:
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -222,9 +272,13 @@ class CamoufoxEngine:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = self._process_response_history(first_response)
try:
@@ -236,15 +290,17 @@ class CamoufoxEngine:
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
body=page_content.encode("utf-8"),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
cookies={
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
**self.adaptor_arguments,
)
page.close()
context.close()
@@ -262,7 +318,10 @@ class CamoufoxEngine:
async def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
async with AsyncCamoufox(**self._get_camoufox_options()) as browser:
@@ -282,7 +341,7 @@ class CamoufoxEngine:
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
await page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
@@ -298,7 +357,7 @@ class CamoufoxEngine:
await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
await page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -309,9 +368,13 @@ class CamoufoxEngine:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = await self._async_process_response_history(first_response)
try:
@@ -323,15 +386,18 @@ class CamoufoxEngine:
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
body=page_content.encode("utf-8"),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
cookies={
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
**self.adaptor_arguments,
)
await page.close()
await context.close()
scrapling/engines/constants.py
+84 -87
View File
@@ -1,92 +1,92 @@
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = {
'font',
'image',
'media',
'beacon',
'object',
'imageset',
'texttrack',
'websocket',
'csp_report',
'stylesheet',
"font",
"image",
"media",
"beacon",
"object",
"imageset",
"texttrack",
"websocket",
"csp_report",
"stylesheet",
}
DEFAULT_STEALTH_FLAGS = (
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--no-pings',
'--incognito',
'--test-type',
'--lang=en-US',
'--mute-audio',
'--no-first-run',
'--disable-sync',
'--hide-scrollbars',
'--disable-logging',
'--start-maximized', # For headless check bypass
'--enable-async-dns',
'--disable-breakpad',
'--disable-infobars',
'--accept-lang=en-US',
'--use-mock-keychain',
'--disable-translate',
'--disable-extensions',
'--disable-voice-input',
'--window-position=0,0',
'--disable-wake-on-wifi',
'--ignore-gpu-blocklist',
'--enable-tcp-fast-open',
'--enable-web-bluetooth',
'--disable-hang-monitor',
'--password-store=basic',
'--disable-cloud-import',
'--disable-default-apps',
'--disable-print-preview',
'--disable-dev-shm-usage',
"--no-pings",
"--incognito",
"--test-type",
"--lang=en-US",
"--mute-audio",
"--no-first-run",
"--disable-sync",
"--hide-scrollbars",
"--disable-logging",
"--start-maximized", # For headless check bypass
"--enable-async-dns",
"--disable-breakpad",
"--disable-infobars",
"--accept-lang=en-US",
"--use-mock-keychain",
"--disable-translate",
"--disable-extensions",
"--disable-voice-input",
"--window-position=0,0",
"--disable-wake-on-wifi",
"--ignore-gpu-blocklist",
"--enable-tcp-fast-open",
"--enable-web-bluetooth",
"--disable-hang-monitor",
"--password-store=basic",
"--disable-cloud-import",
"--disable-default-apps",
"--disable-print-preview",
"--disable-dev-shm-usage",
# '--disable-popup-blocking',
'--metrics-recording-only',
'--disable-crash-reporter',
'--disable-partial-raster',
'--disable-gesture-typing',
'--disable-checker-imaging',
'--disable-prompt-on-repost',
'--force-color-profile=srgb',
'--font-render-hinting=none',
'--no-default-browser-check',
'--aggressive-cache-discard',
'--disable-component-update',
'--disable-cookie-encryption',
'--disable-domain-reliability',
'--disable-threaded-animation',
'--disable-threaded-scrolling',
"--metrics-recording-only",
"--disable-crash-reporter",
"--disable-partial-raster",
"--disable-gesture-typing",
"--disable-checker-imaging",
"--disable-prompt-on-repost",
"--force-color-profile=srgb",
"--font-render-hinting=none",
"--no-default-browser-check",
"--aggressive-cache-discard",
"--disable-component-update",
"--disable-cookie-encryption",
"--disable-domain-reliability",
"--disable-threaded-animation",
"--disable-threaded-scrolling",
# '--disable-reading-from-canvas', # For Firefox
'--enable-simple-cache-backend',
'--disable-background-networking',
'--disable-session-crashed-bubble',
'--enable-surface-synchronization',
'--disable-image-animation-resync',
'--disable-renderer-backgrounding',
'--disable-ipc-flooding-protection',
'--prerender-from-omnibox=disabled',
'--safebrowsing-disable-auto-update',
'--disable-offer-upload-credit-cards',
'--disable-features=site-per-process',
'--disable-background-timer-throttling',
'--disable-new-content-rendering-timeout',
'--run-all-compositor-stages-before-draw',
'--disable-client-side-phishing-detection',
'--disable-backgrounding-occluded-windows',
'--disable-layer-tree-host-memory-pressure',
'--autoplay-policy=no-user-gesture-required',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-blink-features=AutomationControlled',
'--webrtc-ip-handling-policy=disable_non_proxied_udp',
'--disable-component-extensions-with-background-pages',
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
"--enable-simple-cache-backend",
"--disable-background-networking",
"--disable-session-crashed-bubble",
"--enable-surface-synchronization",
"--disable-image-animation-resync",
"--disable-renderer-backgrounding",
"--disable-ipc-flooding-protection",
"--prerender-from-omnibox=disabled",
"--safebrowsing-disable-auto-update",
"--disable-offer-upload-credit-cards",
"--disable-features=site-per-process",
"--disable-background-timer-throttling",
"--disable-new-content-rendering-timeout",
"--run-all-compositor-stages-before-draw",
"--disable-client-side-phishing-detection",
"--disable-backgrounding-occluded-windows",
"--disable-layer-tree-host-memory-pressure",
"--autoplay-policy=no-user-gesture-required",
"--disable-offer-store-unmasked-wallet-cards",
"--disable-blink-features=AutomationControlled",
"--webrtc-ip-handling-policy=disable_non_proxied_udp",
"--disable-component-extensions-with-background-pages",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
"--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance",
"--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
"--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
)
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
@@ -95,13 +95,10 @@ NSTBROWSER_DEFAULT_QUERY = {
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"flags": {"timezone": "BasedOnIp", "screen": "Custom"},
"platform": "linux", # support: windows, mac, linux
"kernel": "chromium", # only support: chromium
"kernelMilestone": "128",
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
scrapling/engines/pw.py
+169 -100
View File
@@ -1,42 +1,46 @@
import json
from scrapling.core._types import (Callable, Dict, Optional,
SelectorWaitStates, Union)
from scrapling.core._types import Callable, Dict, Optional, SelectorWaitStates, Union
from scrapling.core.utils import log, lru_cache
from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
NSTBROWSER_DEFAULT_QUERY)
from scrapling.engines.toolbelt import (Response, StatusText,
async_intercept_route,
check_type_validity, construct_cdp_url,
construct_proxy_dict,
generate_convincing_referer,
generate_headers, intercept_route,
js_bypass_path)
from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
from scrapling.engines.toolbelt import (
Response,
StatusText,
async_intercept_route,
check_type_validity,
construct_cdp_url,
construct_proxy_dict,
generate_convincing_referer,
generate_headers,
intercept_route,
js_bypass_path,
)
class PlaywrightEngine:
def __init__(
self, headless: Union[bool, str] = True,
disable_resources: bool = False,
useragent: Optional[str] = None,
network_idle: bool = False,
timeout: Optional[float] = 30000,
wait: Optional[int] = 0,
page_action: Callable = None,
wait_selector: Optional[str] = None,
locale: Optional[str] = 'en-US',
wait_selector_state: SelectorWaitStates = 'attached',
stealth: bool = False,
real_chrome: bool = False,
hide_canvas: bool = False,
disable_webgl: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None
self,
headless: Union[bool, str] = True,
disable_resources: bool = False,
useragent: Optional[str] = None,
network_idle: bool = False,
timeout: Optional[float] = 30000,
wait: Optional[int] = 0,
page_action: Callable = None,
wait_selector: Optional[str] = None,
locale: Optional[str] = "en-US",
wait_selector_state: SelectorWaitStates = "attached",
stealth: bool = False,
real_chrome: bool = False,
hide_canvas: bool = False,
disable_webgl: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None,
):
"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
@@ -65,7 +69,7 @@ class PlaywrightEngine:
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
self.headless = headless
self.locale = check_type_validity(locale, [str], 'en-US', param_name='locale')
self.locale = check_type_validity(locale, [str], "en-US", param_name="locale")
self.disable_resources = disable_resources
self.network_idle = bool(network_idle)
self.stealth = bool(stealth)
@@ -95,8 +99,8 @@ class PlaywrightEngine:
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
self.harmful_default_args = [
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
'--enable-automation',
'--disable-popup-blocking',
"--enable-automation",
"--disable-popup-blocking",
# '--disable-component-update',
# '--disable-default-apps',
# '--disable-extensions',
@@ -114,12 +118,16 @@ class PlaywrightEngine:
query = NSTBROWSER_DEFAULT_QUERY.copy()
if self.stealth:
flags = self.__set_flags()
query.update({
"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary
})
query.update(
{
"args": dict(
zip(flags, [""] * len(flags))
), # browser args should be a dictionary
}
)
config = {
'config': json.dumps(query),
"config": json.dumps(query),
# 'token': ''
}
cdp_url = construct_cdp_url(cdp_url, config)
@@ -134,17 +142,25 @@ class PlaywrightEngine:
"""Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS
if self.hide_canvas:
flags += ('--fingerprinting-canvas-image-data-noise',)
flags += ("--fingerprinting-canvas-image-data-noise",)
if self.disable_webgl:
flags += ('--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2',)
flags += (
"--disable-webgl",
"--disable-webgl-image-chromium",
"--disable-webgl2",
)
return flags
def __launch_kwargs(self):
"""Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {'headless': self.headless, 'ignore_default_args': self.harmful_default_args, 'channel': 'chrome' if self.real_chrome else 'chromium'}
launch_kwargs = {
"headless": self.headless,
"ignore_default_args": self.harmful_default_args,
"channel": "chrome" if self.real_chrome else "chromium",
}
if self.stealth:
launch_kwargs.update({'args': self.__set_flags(), 'chromium_sandbox': True})
launch_kwargs.update({"args": self.__set_flags(), "chromium_sandbox": True})
return launch_kwargs
@@ -153,22 +169,26 @@ class PlaywrightEngine:
context_kwargs = {
"proxy": self.proxy,
"locale": self.locale,
"color_scheme": 'dark', # Bypasses the 'prefersLightColor' check in creepjs
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
"device_scale_factor": 2,
"extra_http_headers": self.extra_headers if self.extra_headers else {},
"user_agent": self.useragent if self.useragent else generate_headers(browser_mode=True).get('User-Agent'),
"user_agent": self.useragent
if self.useragent
else generate_headers(browser_mode=True).get("User-Agent"),
}
if self.stealth:
context_kwargs.update({
'is_mobile': False,
'has_touch': False,
# I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
'service_workers': 'allow',
'ignore_https_errors': True,
'screen': {'width': 1920, 'height': 1080},
'viewport': {'width': 1920, 'height': 1080},
'permissions': ['geolocation', 'notifications']
})
context_kwargs.update(
{
"is_mobile": False,
"has_touch": False,
# I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
"service_workers": "allow",
"ignore_https_errors": True,
"screen": {"width": 1920, "height": 1080},
"viewport": {"width": 1920, "height": 1080},
"permissions": ["geolocation", "notifications"],
}
)
return context_kwargs
@@ -184,10 +204,16 @@ class PlaywrightEngine:
# https://arh.antoinevastel.com/bots/areyouheadless/
# https://prescience-data.github.io/execution-monitor.html
return tuple(
js_bypass_path(script) for script in (
js_bypass_path(script)
for script in (
# Order is important
'webdriver_fully.js', 'window_chrome.js', 'navigator_plugins.js', 'pdf_viewer.js',
'notification_permission.js', 'screen_props.js', 'playwright_fingerprint.js'
"webdriver_fully.js",
"window_chrome.js",
"navigator_plugins.js",
"pdf_viewer.js",
"notification_permission.js",
"screen_props.js",
"playwright_fingerprint.js",
)
)
@@ -200,19 +226,30 @@ class PlaywrightEngine:
while current_request:
try:
current_response = current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments
))
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text="",
body=b"",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=current_response.all_headers()
if current_response
else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
@@ -232,19 +269,30 @@ class PlaywrightEngine:
while current_request:
try:
current_response = await current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments
))
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text="",
body=b"",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=await current_response.all_headers()
if current_response
else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
@@ -262,6 +310,7 @@ class PlaywrightEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
from playwright.sync_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.sync_api import sync_playwright
@@ -273,7 +322,10 @@ class PlaywrightEngine:
def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
with sync_playwright() as p:
@@ -304,7 +356,7 @@ class PlaywrightEngine:
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
@@ -320,7 +372,7 @@ class PlaywrightEngine:
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -331,9 +383,13 @@ class PlaywrightEngine:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = self._process_response_history(first_response)
try:
@@ -345,15 +401,17 @@ class PlaywrightEngine:
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
body=page_content.encode("utf-8"),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
cookies={
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
**self.adaptor_arguments,
)
page.close()
context.close()
@@ -366,6 +424,7 @@ class PlaywrightEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
from playwright.async_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.async_api import async_playwright
@@ -377,7 +436,10 @@ class PlaywrightEngine:
async def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
async with async_playwright() as p:
@@ -408,7 +470,7 @@ class PlaywrightEngine:
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
await page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
@@ -424,7 +486,7 @@ class PlaywrightEngine:
await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
await page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -435,9 +497,13 @@ class PlaywrightEngine:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = await self._async_process_response_history(first_response)
try:
@@ -449,15 +515,18 @@ class PlaywrightEngine:
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
body=page_content.encode("utf-8"),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
cookies={
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
**self.adaptor_arguments,
)
await page.close()
await context.close()
scrapling/engines/static.py
+57 -25
View File
@@ -10,8 +10,14 @@ from .toolbelt import Response, generate_convincing_referer, generate_headers
@lru_cache(2, typed=True) # Singleton easily
class StaticEngine:
def __init__(
self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None
self,
url: str,
proxy: Optional[str] = None,
stealthy_headers: bool = True,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None,
retries: Optional[int] = 3,
adaptor_arguments: Tuple = None,
):
"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
@@ -47,14 +53,22 @@ class StaticEngine:
if self.stealth:
extra_headers = generate_headers(browser_mode=False)
# Don't overwrite user supplied headers
extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}
extra_headers = {
key: value
for key, value in extra_headers.items()
if key.lower() not in headers_keys
}
headers.update(extra_headers)
if 'referer' not in headers_keys:
headers.update({'referer': generate_convincing_referer(self.url)})
if "referer" not in headers_keys:
headers.update({"referer": generate_convincing_referer(self.url)})
elif 'user-agent' not in headers_keys:
headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
elif "user-agent" not in headers_keys:
headers["User-Agent"] = generate_headers(browser_mode=False).get(
"User-Agent"
)
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."
)
return headers
@@ -70,25 +84,43 @@ class StaticEngine:
body=response.content,
status=response.status_code,
reason=response.reason_phrase,
encoding=response.encoding or 'utf-8',
encoding=response.encoding or "utf-8",
cookies=dict(response.cookies),
headers=dict(response.headers),
request_headers=dict(response.request.headers),
method=response.request.method,
history=[self._prepare_response(redirection) for redirection in response.history],
**self.adaptor_arguments
history=[
self._prepare_response(redirection) for redirection in response.history
],
**self.adaptor_arguments,
)
def _make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {}))
with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
request = getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
headers = self._headers_job(kwargs.pop("headers", {}))
with httpx.Client(
proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)
) as client:
request = getattr(client, method)(
url=self.url,
headers=headers,
follow_redirects=self.follow_redirects,
timeout=self.timeout,
**kwargs,
)
return self._prepare_response(request)
async def _async_make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {}))
async with httpx.AsyncClient(proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)) as client:
request = await getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
headers = self._headers_job(kwargs.pop("headers", {}))
async with httpx.AsyncClient(
proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)
) as client:
request = await getattr(client, method)(
url=self.url,
headers=headers,
follow_redirects=self.follow_redirects,
timeout=self.timeout,
**kwargs,
)
return self._prepare_response(request)
def get(self, **kwargs: Dict) -> Response:
@@ -97,7 +129,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return self._make_request('get', **kwargs)
return self._make_request("get", **kwargs)
async def async_get(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP GET request for you but with some added flavors.
@@ -105,7 +137,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return await self._async_make_request('get', **kwargs)
return await self._async_make_request("get", **kwargs)
def post(self, **kwargs: Dict) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
@@ -113,7 +145,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return self._make_request('post', **kwargs)
return self._make_request("post", **kwargs)
async def async_post(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP POST request for you but with some added flavors.
@@ -121,7 +153,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return await self._async_make_request('post', **kwargs)
return await self._async_make_request("post", **kwargs)
def delete(self, **kwargs: Dict) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
@@ -129,7 +161,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return self._make_request('delete', **kwargs)
return self._make_request("delete", **kwargs)
async def async_delete(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP DELETE request for you but with some added flavors.
@@ -137,7 +169,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return await self._async_make_request('delete', **kwargs)
return await self._async_make_request("delete", **kwargs)
def put(self, **kwargs: Dict) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
@@ -145,7 +177,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return self._make_request('put', **kwargs)
return self._make_request("put", **kwargs)
async def async_put(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP PUT request for you but with some added flavors.
@@ -153,4 +185,4 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return await self._async_make_request('put', **kwargs)
return await self._async_make_request("put", **kwargs)
scrapling/engines/toolbelt/__init__.py
+16 -6
View File
@@ -1,6 +1,16 @@
from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,
check_type_validity, get_variable_name)
from .fingerprints import (generate_convincing_referer, generate_headers,
get_os_name)
from .navigation import (async_intercept_route, construct_cdp_url,
construct_proxy_dict, intercept_route, js_bypass_path)
from .custom import (
BaseFetcher,
Response,
StatusText,
check_if_engine_usable,
check_type_validity,
get_variable_name,
)
from .fingerprints import generate_convincing_referer, generate_headers, get_os_name
from .navigation import (
async_intercept_route,
construct_cdp_url,
construct_proxy_dict,
intercept_route,
js_bypass_path,
)
scrapling/engines/toolbelt/custom.py
+167 -95
View File
@@ -1,11 +1,20 @@
"""
Functions related to custom types or type checking
"""
import inspect
from email.message import Message
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
Type, Union)
from scrapling.core._types import (
Any,
Callable,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem
@@ -13,7 +22,12 @@ from scrapling.parser import Adaptor, SQLiteStorageSystem
class ResponseEncoding:
__DEFAULT_ENCODING = "utf-8"
__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
__ISO_8859_1_CONTENT_TYPES = {
"text/plain",
"text/html",
"text/css",
"text/javascript",
}
@classmethod
@lru_cache(maxsize=128)
@@ -27,19 +41,21 @@ class ResponseEncoding:
"""
# Create a Message object and set the Content-Type header then get the content type and parameters
msg = Message()
msg['content-type'] = header_value
msg["content-type"] = header_value
content_type = msg.get_content_type()
params = dict(msg.get_params(failobj=[]))
# Remove the content-type from params if present somehow
params.pop('content-type', None)
params.pop("content-type", None)
return content_type, params
@classmethod
@lru_cache(maxsize=128)
def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
"""Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order:
@@ -72,7 +88,9 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING
if encoding:
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text
_ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
return encoding
return cls.__DEFAULT_ENCODING
@@ -84,9 +102,22 @@ class ResponseEncoding:
class Response(Adaptor):
"""This class is returned by all engines as a way to unify response type between different libraries."""
def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):
automatch_domain = adaptor_arguments.pop('automatch_domain', None)
def __init__(
self,
url: str,
text: str,
body: bytes,
status: int,
reason: str,
cookies: Dict,
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**adaptor_arguments: Dict,
):
automatch_domain = adaptor_arguments.pop("automatch_domain", None)
self.status = status
self.reason = reason
self.cookies = cookies
@@ -94,11 +125,19 @@ class Response(Adaptor):
self.request_headers = request_headers
self.history = history or []
encoding = ResponseEncoding.get_value(encoding, text)
super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
super().__init__(
text=text,
body=body,
url=automatch_domain or url,
encoding=encoding,
**adaptor_arguments,
)
# For back-ward compatibility
self.adaptor = self
# For easier debugging while working from a Python shell
log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
# def __repr__(self):
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
@@ -113,16 +152,26 @@ class BaseFetcher:
storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None
parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',) # Left open for the user
parser_keywords: Tuple = (
"huge_tree",
"auto_match",
"storage",
"keep_cdata",
"storage_args",
"keep_comments",
"automatch_domain",
) # Left open for the user
def __init__(self, *args, **kwargs):
# For backward-compatibility before 0.2.99
args_str = ", ".join(args) or ''
kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
args_str = ", ".join(args) or ""
kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) or ""
if args_str:
args_str += ', '
args_str += ", "
log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
log.warning(
f"This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching"
)
pass
@classmethod
@@ -150,12 +199,18 @@ class BaseFetcher:
setattr(cls, key, value)
else:
# Yup, no fun allowed LOL
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
else:
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
if not kwargs:
raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
@classmethod
def _generate_parser_arguments(cls) -> Dict:
@@ -167,13 +222,15 @@ class BaseFetcher:
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
storage=cls.storage,
storage_args=cls.storage_args
storage_args=cls.storage_args,
)
if cls.automatch_domain:
if type(cls.automatch_domain) is not str:
log.warning('[Ignored] The argument "automatch_domain" must be of string type')
log.warning(
'[Ignored] The argument "automatch_domain" must be of string type'
)
else:
parser_arguments.update({'automatch_domain': cls.automatch_domain})
parser_arguments.update({"automatch_domain": cls.automatch_domain})
return parser_arguments
@@ -181,72 +238,75 @@ class BaseFetcher:
class StatusText:
"""A class that gets the status text of response status code.
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
"""
_phrases = MappingProxyType({
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required"
})
_phrases = MappingProxyType(
{
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
}
)
@classmethod
@lru_cache(maxsize=128)
@@ -265,20 +325,26 @@ def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
if hasattr(engine, "fetch"):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError(
"Engine class must have a callable method 'fetch' with the first argument used for the url."
)
else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
raise TypeError(
"Invalid engine class! Engine class must have a callable method 'fetch'"
)
else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
raise TypeError(
"Invalid engine class! Engine class must have the method 'fetch'"
)
def get_variable_name(var: Any) -> Optional[str]:
@@ -293,7 +359,13 @@ def get_variable_name(var: Any) -> Optional[str]:
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
def check_type_validity(
variable: Any,
valid_types: Union[List[Type], None],
default_value: Any = None,
critical: bool = False,
param_name: Optional[str] = None,
) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
@@ -316,7 +388,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
log.error(f"[Ignored] {error_msg}")
return default_value
# If no valid_types specified and variable has a value, return it
@@ -329,7 +401,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
log.error(f"[Ignored] {error_msg}")
return default_value
return variable
scrapling/engines/toolbelt/fingerprints.py
+13 -13
View File
@@ -23,7 +23,7 @@ def generate_convincing_referer(url: str) -> str:
:return: Google's search URL of the domain name
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
return f"https://www.google.com/search?q={website_name}"
@lru_cache(1, typed=True)
@@ -35,11 +35,11 @@ def get_os_name() -> Union[str, None]:
#
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
"Linux": "linux",
"Darwin": "macos",
"Windows": "windows",
# For the future? because why not
'iOS': 'ios',
"iOS": "ios",
}.get(os_name)
@@ -50,9 +50,9 @@ def generate_suitable_fingerprint() -> Fingerprint:
:return: `Fingerprint` object
"""
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
browser=[Browser(name="chrome", min_version=128)],
os=get_os_name(), # None is ignored
device='desktop'
device="desktop",
).generate()
@@ -67,15 +67,15 @@ def generate_headers(browser_mode: bool = False) -> Dict:
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=130)],
browser=[Browser(name="chrome", min_version=130)],
os=os_name, # None is ignored
device='desktop'
device="desktop",
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
Browser(name="chrome", min_version=120),
Browser(name="firefox", min_version=120),
Browser(name="edge", min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
return HeaderGenerator(browser=browsers, device="desktop").generate()
scrapling/engines/toolbelt/navigation.py
+29 -14
View File
@@ -1,6 +1,7 @@
"""
Functions related to files and URLs
"""
import os
from urllib.parse import urlencode, urlparse
@@ -19,7 +20,9 @@ def intercept_route(route: Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
route.abort()
else:
route.continue_()
@@ -32,7 +35,9 @@ async def async_intercept_route(route: async_Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
await route.abort()
else:
await route.continue_()
@@ -50,23 +55,33 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict
proxy = urlparse(proxy_string)
try:
return {
'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
'username': proxy.username or '',
'password': proxy.password or '',
"server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",
"username": proxy.username or "",
"password": proxy.password or "",
}
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError('The proxy argument\'s string is in invalid format!')
raise TypeError("The proxy argument's string is in invalid format!")
elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', )
if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
valid_keys = (
"server",
"username",
"password",
)
if all(key in valid_keys for key in proxy_string.keys()) and not any(
key not in valid_keys for key in proxy_string.keys()
):
return proxy_string
else:
raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
raise TypeError(
f"A proxy dictionary must have only these keys: {valid_keys}"
)
else:
raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
raise TypeError(
f"Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!"
)
# The default value for proxy in Playwright's source is `None`
return None
@@ -84,7 +99,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
parsed = urlparse(cdp_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
if parsed.scheme not in ("ws", "wss"):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
@@ -93,8 +108,8 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
if not path.startswith("/"):
path = "/" + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
@@ -118,4 +133,4 @@ def js_bypass_path(filename: str) -> str:
:return: The full path of the JS file.
"""
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
return os.path.join(current_directory, "bypasses", filename)