style: Fix all mypy errors and add type hints to untyped function bodies

**Resolved all 65 mypy errors across 14 files and added type annotations to all previously untyped function bodies. Final result: 0 errors with --check-untyped-defs enabled, all 454 tests pass.**

`scrapling/core/_types.py`

  - Removed broken Self = object fallback — now requires typing_extensions for Python < 3.11

`scrapling/core/storage.py`

  - Fixed str/bytes mismatch in _get_hash() — used separate _identifier_bytes variable instead of reassigning from str to bytes

`scrapling/core/custom_types.py`

  - split() return type: Union[List, "TextHandlers"] → list[Any] (avoids LSP violation with parent list[str])
  - format() kwargs: **kwargs: str → **kwargs: object (matches parent str.format signature)
  - AttributesHandler.__init__: Added mapping: Any = None, **kwargs: Any and -> None
  - json_string property: Added -> bytes return type

`scrapling/core/mixins.py`

  - Changed self: "Selector" to self: Any on all mixin methods (mypy can't handle forward-reference self types on non-subclass mixins)
  - Added Dict[str, int] annotation for counter variable
  - Removed unused TYPE_CHECKING / Selector imports

`scrapling/parser.py (~30 errors)`

  - Added body: str | bytes pre-annotation for dual-type if/elif assignment
  - Used Dict[str, Any] kwargs dict for HTMLParser(...) to bypass incomplete lxml stubs missing default_doctype
  - Changed base_url=url or None → base_url=url or "" (avoids str | None vs str | bytes)
  - bool(adaptive) to guarantee bool type for __adaptive_enabled
  - Declared __text: Optional[TextHandler], __tag: Optional[str], __attributes: Optional[AttributesHandler] at top of __init__
  - cast(List, ...) for all XPath() call results (_find_all_elements, _find_all_elements_with_spaces)
  - Added Dict[float, List[Any]] for score_table, Dict[str, Any] for attributes
  - Changed score, checks = 0, 0 → score: float = 0; checks: int = 0 (two locations)
  - Renamed target → target_element in save() to avoid variable redefinition with different types
  - Wrapped node_text.clean() / .lower() in TextHandler(...) to preserve type

`scrapling/engines/_browsers/_page.py`

  - Added PageInfo[SyncPage] | PageInfo[AsyncPage] union type annotation to page_info variable

`scrapling/engines/_browsers/_validators.py`

  - Convert method_kwargs (TypedDict) to plain Dict[str, Any] before dynamic key access

`scrapling/engines/_browsers/_base.py`

  - Added _config declaration to BaseSessionMixin
  - Used cast(StealthConfig, self._config) in __generate_stealth_options to access stealth-only attributes
  - Added Tuple[str, ...] annotation for flags
  - Removed redundant narrower StealthConfig type annotation on self._config in StealthySessionMixin.__validate__
  - Widened SyncSession and AsyncSession fields (playwright, context, browser) to Any to support both playwright and patchright types
  - Added -> None to both start() methods

`scrapling/engines/_browsers/_stealth.py`

  - Added Optional, ProxyType imports
  - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops
  - Annotated outer_box: Any at first declaration, removed duplicate type annotations in subsequent branches
  - Added -> None to sync and async start()
  - Added config: Any parameter type to _initialize_context
  - Removed redundant self.context: AsyncBrowserContext re-annotations in conditional branches

`scrapling/engines/_browsers/_controllers.py`

  - Added Optional, ProxyType imports
  - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops
  - Added -> None to async start()
  - Removed redundant self.context: AsyncBrowserContext re-annotations

`scrapling/spiders/request.py`

  - Added Optional import, typed _fp: Optional[bytes] = None
  - Removed redundant body: bytes re-annotation

`scrapling/spiders/session.py`

  - Used separate client variable instead of reassigning session = session._client (avoids type incompatibility and fixes a bug where session._make_request was called instead of client._make_request)
  - Added -> None to SessionManager.__init__

`scrapling/engines/toolbelt/convertor.py`

  - Added list[Response] annotation for history in both sync/async methods

`scrapling/engines/static.py`

  - FetcherClient.__init__ and AsyncFetcherClient.__init__: Added **kwargs: Any and -> None

`scrapling/core/shell.py`

  - Wrapped re_sub(...) result in TextHandler(...) to maintain correct type
  - Added -> None to CurlParser.__init__
  - Added full type signature to create_wrapper, replaced wrapper.__signature__ = ... with setattr(wrapper, "__signature__", ...) to satisfy mypy
  - Added Callable to imports
This commit is contained in:
Karim shoair
2026-02-07 16:30:00 +02:00
parent 5f557a7f9f
commit 5ec929435b
15 changed files with 133 additions and 149 deletions
+14 -14
View File
@@ -13,7 +13,7 @@ from patchright.sync_api import sync_playwright
from patchright.async_api import async_playwright
from scrapling.core.utils import log
from scrapling.core._types import Any, Unpack
from scrapling.core._types import Any, Optional, ProxyType, Unpack
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
@@ -78,7 +78,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
self.__validate__(**kwargs)
super().__init__()
def start(self):
def start(self) -> None:
"""Create a browser for this instance and context."""
if not self.playwright:
self.playwright = sync_playwright().start()
@@ -146,7 +146,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
page.wait_for_timeout(500)
outer_box = {}
outer_box: Any = {}
iframe = page.frame(url=__CF_PATTERN__)
if iframe is not None:
self._wait_for_page_stability(iframe, True, False)
@@ -156,14 +156,14 @@ class StealthySession(SyncSession, StealthySessionMixin):
# Double-checking that the iframe is loaded
page.wait_for_timeout(500)
outer_box: Any = iframe.frame_element().bounding_box()
outer_box = iframe.frame_element().bounding_box()
if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
log.info("Cloudflare captcha is solved")
return
outer_box: Any = page.locator(box_selector).last.bounding_box()
outer_box = page.locator(box_selector).last.bounding_box()
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -223,6 +223,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
@@ -335,7 +336,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
self.__validate__(**kwargs)
super().__init__(max_pages=self._config.max_pages)
async def start(self):
async def start(self) -> None:
"""Create a browser for this instance and context."""
if not self.playwright:
self.playwright = await async_playwright().start()
@@ -344,16 +345,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
if not self._config.proxy_rotator:
assert self.browser is not None
self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options)
self.context = await self.browser.new_context(**self._context_options)
elif self._config.proxy_rotator:
self.browser = await self.playwright.chromium.launch(**self._browser_options)
else:
persistent_options = (
self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
)
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**persistent_options
)
self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
if self.context:
self.context = await self._initialize_context(self._config, self.context)
@@ -367,7 +366,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else:
raise RuntimeError("Session has been already started")
async def _initialize_context(self, config, ctx: AsyncBrowserContext) -> AsyncBrowserContext:
async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext:
"""Initialize the browser context."""
for script in _compiled_stealth_scripts():
await ctx.add_init_script(script=script)
@@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
await page.wait_for_timeout(500)
outer_box = {}
outer_box: Any = {}
iframe = page.frame(url=__CF_PATTERN__)
if iframe is not None:
await self._wait_for_page_stability(iframe, True, False)
@@ -414,14 +413,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
# Double-checking that the iframe is loaded
await page.wait_for_timeout(500)
outer_box: Any = await (await iframe.frame_element()).bounding_box()
outer_box = await (await iframe.frame_element()).bounding_box()
if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
log.info("Cloudflare captcha is solved")
return
outer_box: Any = await page.locator(box_selector).last.bounding_box()
outer_box = await page.locator(box_selector).last.bounding_box()
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -482,6 +481,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else: