freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Making all the codebase acceptable by PyRight

Browse Source 
Also fixes #97
This commit is contained in:
Karim shoair
2025-10-05 04:03:39 +03:00
parent e149c715dd
commit debe03256b
21 changed files with 306 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+28 -11
View File
@@ -7,14 +7,12 @@ from playwright.async_api import (
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
)
from camoufox.utils import (
launch_options as generate_launch_options,
installed_verstr as camoufox_version,
)
from camoufox.pkgman import installed_verstr as camoufox_version
from camoufox.utils import launch_options as generate_launch_options
from ._page import PageInfo, PagePool
from scrapling.parser import Selector
from scrapling.core._types import Dict, Optional
from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING
from scrapling.engines.toolbelt.fingerprints import get_os_name
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
@@ -41,6 +39,7 @@ class SyncSession:
"""Get a new page to use"""
# No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc.
assert self.context is not None, "Browser context not initialized"
page = self.context.new_page()
page.set_default_navigation_timeout(timeout)
page.set_default_timeout(timeout)
@@ -65,11 +64,14 @@ class SyncSession:
}
class AsyncSession(SyncSession):
class AsyncSession:
def __init__(self, max_pages: int = 1):
super().__init__(max_pages)
self.max_pages = max_pages
self.page_pool = PagePool(max_pages)
self._max_wait_for_page = 60
self.playwright: Optional[AsyncPlaywright] = None
self.context: Optional[AsyncBrowserContext] = None
self._closed = False
self._lock = Lock()
async def _get_page(
@@ -79,6 +81,9 @@ class AsyncSession(SyncSession):
disable_resources: bool,
) -> PageInfo: # pragma: no cover
"""Get a new page to use"""
if TYPE_CHECKING:
assert self.context is not None, "Browser context not initialized"
async with self._lock:
# If we're at max capacity after cleanup, wait for busy pages to finish
if self.page_pool.pages_count >= self.max_pages:
@@ -92,6 +97,7 @@ class AsyncSession(SyncSession):
f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period"
)
assert self.context is not None, "Browser context not initialized"
page = await self.context.new_page()
page.set_default_navigation_timeout(timeout)
page.set_default_timeout(timeout)
@@ -107,6 +113,14 @@ class AsyncSession(SyncSession):
return self.page_pool.add_page(page)
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
"total_pages": self.page_pool.pages_count,
"busy_pages": self.page_pool.busy_count,
"max_pages": self.max_pages,
}
class DynamicSessionMixin:
def __validate__(self, **params):
@@ -139,6 +153,9 @@ class DynamicSessionMixin:
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
if TYPE_CHECKING:
assert isinstance(self.proxy, tuple)
if not self.cdp_url:
# `launch_options` is used with persistent context
self.launch_options = dict(
@@ -175,7 +192,7 @@ class DynamicSessionMixin:
class StealthySessionMixin:
def __validate__(self, **params):
config = validate(params, model=CamoufoxConfig)
config: CamoufoxConfig = validate(params, model=CamoufoxConfig)
self.max_pages = config.max_pages
self.headless = config.headless
@@ -209,10 +226,10 @@ class StealthySessionMixin:
def __initiate_browser_options__(self):
"""Initiate browser options."""
self.launch_options = generate_launch_options(
self.launch_options: Dict[str, Any] = generate_launch_options(
**{
"geoip": self.geoip,
"proxy": dict(self.proxy) if self.proxy else self.proxy,
"proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy,
"addons": self.addons,
"exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
"headless": self.headless,
@@ -232,7 +249,7 @@ class StealthySessionMixin:
"browser.cache.disk_cache_ssl": True,
"browser.cache.disk.smart_size.enabled": True,
},
**self.additional_args,
**cast(Dict, self.additional_args),
}
)
scrapling/engines/_browsers/_camoufox.py
+12 -5
View File
@@ -26,6 +26,7 @@ from scrapling.core._types import (
List,
Optional,
Callable,
TYPE_CHECKING,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.convertor import (
@@ -205,7 +206,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
self._closed = True
@staticmethod
def _get_page_content(page: Page) -> str | None:
def _get_page_content(page: Page) -> str:
"""
A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from.
@@ -217,6 +218,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
except PlaywrightError:
page.wait_for_timeout(1000)
continue
return "" # pyright: ignore
def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
@@ -502,8 +504,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start()
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
self.playwright: AsyncPlaywright | None = await async_playwright().start()
self.context: AsyncBrowserContext | None = await self.playwright.firefox.launch_persistent_context(
**self.launch_options
)
@@ -511,7 +513,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
await self.context.add_init_script(path=self.init_script)
if self.cookies:
await self.context.add_cookies(self.cookies)
await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType]
async def __aenter__(self):
await self.__create__()
@@ -536,7 +538,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
self._closed = True
@staticmethod
async def _get_page_content(page: async_Page) -> str | None:
async def _get_page_content(page: async_Page) -> str:
"""
A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from.
@@ -548,6 +550,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
except PlaywrightError:
await page.wait_for_timeout(1000)
continue
return "" # pyright: ignore
async def _solve_cloudflare(self, page: async_Page):
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
@@ -679,6 +682,10 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
if TYPE_CHECKING:
if not isinstance(page_info.page, async_Page):
raise TypeError
try:
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
scrapling/engines/_browsers/_config_tools.py
+1 -1
View File
@@ -62,7 +62,7 @@ def _set_flags(hide_canvas, disable_webgl): # pragma: no cover
@lru_cache(2, typed=True)
def _launch_kwargs(
headless,
proxy,
proxy: Tuple,
locale,
extra_headers,
useragent,
scrapling/engines/_browsers/_controllers.py
+14 -7
View File
@@ -10,6 +10,7 @@ from playwright.async_api import (
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
Locator as AsyncLocator,
Page as async_Page,
)
from patchright.sync_api import sync_playwright as sync_patchright
from patchright.async_api import async_playwright as async_patchright
@@ -18,10 +19,12 @@ from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate_fetch as _validate
from scrapling.core._types import (
Any,
Dict,
List,
Optional,
Callable,
TYPE_CHECKING,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.convertor import (
@@ -30,7 +33,7 @@ from scrapling.engines.toolbelt.convertor import (
)
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
_UNSET = object()
_UNSET: Any = object()
class DynamicSession(DynamicSessionMixin, SyncSession):
@@ -154,7 +157,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
"""Create a browser for this instance and context."""
sync_context = sync_patchright if self.stealth else sync_playwright
self.playwright: Playwright = sync_context().start()
self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
@@ -187,7 +190,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
if self.playwright:
self.playwright.stop()
self.playwright = None
self.playwright = None # pyright: ignore
self._closed = True
@@ -399,7 +402,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
"""Create a browser for this instance and context."""
async_context = async_patchright if self.stealth else async_playwright
self.playwright: AsyncPlaywright = await async_context().start()
self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
@@ -413,7 +416,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
await self.context.add_init_script(path=self.init_script)
if self.cookies:
await self.context.add_cookies(self.cookies)
await self.context.add_cookies(self.cookies) # pyright: ignore
async def __aenter__(self):
await self.__create__()
@@ -429,11 +432,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
if self.context:
await self.context.close()
self.context = None
self.context = None # pyright: ignore
if self.playwright:
await self.playwright.stop()
self.playwright = None
self.playwright = None # pyright: ignore
self._closed = True
@@ -506,6 +509,10 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
if TYPE_CHECKING:
if not isinstance(page_info.page, async_Page):
raise TypeError
try:
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
scrapling/engines/_browsers/_validators.py
+28 -11
View File
@@ -11,7 +11,10 @@ from scrapling.core._types import (
Tuple,
Optional,
Callable,
Iterable,
SelectorWaitStates,
cast,
overload,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
@@ -73,7 +76,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
stealth: bool = False
wait: Seconds = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
@@ -81,11 +84,11 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
cookies: Optional[Iterable[Dict]] = None
network_idle: bool = False
load_dom: bool = True
wait_selector_state: SelectorWaitStates = "attached"
selector_config: Optional[Dict] = None
selector_config: Optional[Dict] = {}
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -125,15 +128,15 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
wait_selector_state: SelectorWaitStates = "attached"
cookies: Optional[List[Dict]] = None
cookies: Optional[Iterable[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
selector_config: Optional[Dict] = None
additional_args: Optional[Dict] = None
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -177,7 +180,7 @@ class FetchConfig(Struct, kw_only=True):
network_idle: bool = False
load_dom: bool = True
solve_cloudflare: bool = False
selector_config: Optional[Dict] = {}
selector_config: Dict = {}
def to_dict(self):
return {f: getattr(self, f) for f in self.__struct_fields__}
@@ -198,7 +201,7 @@ class _fetch_params:
network_idle: bool
load_dom: bool
solve_cloudflare: bool
selector_config: Optional[Dict]
selector_config: Dict
def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
@@ -212,7 +215,7 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
result[arg] = session_value
if overrides:
overrides = validate(overrides, FetchConfig).to_dict()
overrides = cast(FetchConfig, validate(overrides, FetchConfig)).to_dict()
overrides.update(result)
return _fetch_params(**overrides)
@@ -222,7 +225,21 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
return _fetch_params(**result)
def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
@overload
def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
@overload
def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
@overload
def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ...
def validate(
params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig]
) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
try:
return convert(params, model)
except ValidationError as e:
scrapling/engines/static.py
+4 -4
View File
@@ -182,7 +182,7 @@ class FetcherSession:
return headers
def __enter__(self):
def __enter__(self) -> "FetcherClient":
"""Creates and returns a new synchronous Fetcher Session"""
if self._curl_session:
raise RuntimeError(
@@ -197,7 +197,7 @@ class FetcherSession:
)
self._curl_session = CurlSession()
return self
return cast("FetcherClient", self)
def __exit__(self, exc_type, exc_val, exc_tb):
"""Closes the active synchronous session managed by this instance, if any."""
@@ -205,7 +205,7 @@ class FetcherSession:
self._curl_session.close()
self._curl_session = None
async def __aenter__(self):
async def __aenter__(self) -> "AsyncFetcherClient":
"""Creates and returns a new asynchronous Session."""
if self._async_curl_session:
raise RuntimeError(
@@ -220,7 +220,7 @@ class FetcherSession:
)
self._async_curl_session = AsyncCurlSession()
return self
return cast("AsyncFetcherClient", self)
async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Closes the active asynchronous session managed by this instance, if any."""
scrapling/engines/toolbelt/convertor.py
+6 -4
View File
@@ -58,7 +58,8 @@ class ResponseFactory:
"encoding": cls.__extract_browser_encoding(
current_response.headers.get("content-type", "")
)
or "utf-8",
if current_response
else "utf-8",
"cookies": tuple(),
"headers": current_response.all_headers() if current_response else {},
"request_headers": current_request.all_headers(),
@@ -161,7 +162,8 @@ class ResponseFactory:
"encoding": cls.__extract_browser_encoding(
current_response.headers.get("content-type", "")
)
or "utf-8",
if current_response
else "utf-8",
"cookies": tuple(),
"headers": await current_response.all_headers() if current_response else {},
"request_headers": await current_request.all_headers(),
@@ -255,8 +257,8 @@ class ResponseFactory:
"encoding": response.encoding or "utf-8",
"cookies": dict(response.cookies),
"headers": dict(response.headers),
"request_headers": dict(response.request.headers),
"method": response.request.method,
"request_headers": dict(response.request.headers) if response.request else {},
"method": response.request.method if response.request else "GET",
"history": response.history, # https://github.com/lexiforest/curl_cffi/issues/82
**parser_arguments,
}
scrapling/engines/toolbelt/custom.py
+6 -9
View File
@@ -8,6 +8,7 @@ from scrapling.core.utils import log
from scrapling.core._types import (
Any,
Dict,
cast,
List,
Optional,
Tuple,
@@ -30,10 +31,10 @@ class Response(Selector):
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**selector_config: Dict,
history: List | None = None,
**selector_config: Any,
):
adaptive_domain = selector_config.pop("adaptive_domain", None)
adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))
self.status = status
self.reason = reason
self.cookies = cookies
@@ -58,7 +59,7 @@ class BaseFetcher:
keep_cdata: Optional[bool] = False
storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False
adaptive_domain: Optional[str] = None
adaptive_domain: str = ""
parser_keywords: Tuple = (
"huge_tree",
"adaptive",
@@ -124,12 +125,8 @@ class BaseFetcher:
adaptive=cls.adaptive,
storage=cls.storage,
storage_args=cls.storage_args,
adaptive_domain=cls.adaptive_domain,
)
if cls.adaptive_domain:
if not isinstance(cls.adaptive_domain, str):
log.warning('[Ignored] The argument "adaptive_domain" must be of string type')
else:
parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
return parser_arguments
scrapling/engines/toolbelt/fingerprints.py
+17 -10
View File
@@ -8,9 +8,10 @@ from platform import system as platform_system
from tldextract import extract
from browserforge.headers import Browser, HeaderGenerator
from scrapling.core._types import Dict, Optional
from scrapling.core._types import Dict, Literal
__OS_NAME__ = platform_system()
OSName = Literal["linux", "macos", "windows"]
@lru_cache(10, typed=True)
@@ -28,16 +29,20 @@ def generate_convincing_referer(url: str) -> str:
@lru_cache(1, typed=True)
def get_os_name() -> Optional[str]:
"""Get the current OS name in the same format needed for browserforge
def get_os_name() -> OSName | None:
"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
:return: Current OS name or `None` otherwise
"""
return {
"Linux": "linux",
"Darwin": "macos",
"Windows": "windows",
}.get(__OS_NAME__)
match __OS_NAME__:
case "Linux":
return "linux"
case "Darwin":
return "macos"
case "Windows":
return "windows"
case _:
return None
def generate_headers(browser_mode: bool = False) -> Dict:
@@ -58,8 +63,10 @@ def generate_headers(browser_mode: bool = False) -> Dict:
Browser(name="edge", min_version=130),
]
)
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
if os_name:
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
else:
return HeaderGenerator(browser=browsers, device="desktop").generate()
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
scrapling/engines/toolbelt/navigation.py
+11 -3
View File
@@ -11,7 +11,7 @@ from msgspec import Struct, structs, convert, ValidationError
from playwright.sync_api import Route
from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Tuple
from scrapling.core._types import Dict, Tuple, overload, Literal
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
@@ -49,7 +49,15 @@ async def async_intercept_route(route: async_Route):
await route.continue_()
def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]:
@overload
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[True]) -> Tuple: ...
@overload
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[False] = False) -> Dict: ...
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: bool = False) -> Dict | Tuple:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy
@@ -83,7 +91,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->
except ValidationError as e:
raise TypeError(f"Invalid proxy dictionary: {e}")
return None
raise TypeError(f"Invalid proxy string: {proxy_string}")
@lru_cache(10, typed=True)