style: Fix all mypy errors and add type hints to untyped function bodies
**Resolved all 65 mypy errors across 14 files and added type annotations to all previously untyped function bodies. Final result: 0 errors with --check-untyped-defs enabled, all 454 tests pass.** `scrapling/core/_types.py` - Removed broken Self = object fallback — now requires typing_extensions for Python < 3.11 `scrapling/core/storage.py` - Fixed str/bytes mismatch in _get_hash() — used separate _identifier_bytes variable instead of reassigning from str to bytes `scrapling/core/custom_types.py` - split() return type: Union[List, "TextHandlers"] → list[Any] (avoids LSP violation with parent list[str]) - format() kwargs: **kwargs: str → **kwargs: object (matches parent str.format signature) - AttributesHandler.__init__: Added mapping: Any = None, **kwargs: Any and -> None - json_string property: Added -> bytes return type `scrapling/core/mixins.py` - Changed self: "Selector" to self: Any on all mixin methods (mypy can't handle forward-reference self types on non-subclass mixins) - Added Dict[str, int] annotation for counter variable - Removed unused TYPE_CHECKING / Selector imports `scrapling/parser.py (~30 errors)` - Added body: str | bytes pre-annotation for dual-type if/elif assignment - Used Dict[str, Any] kwargs dict for HTMLParser(...) to bypass incomplete lxml stubs missing default_doctype - Changed base_url=url or None → base_url=url or "" (avoids str | None vs str | bytes) - bool(adaptive) to guarantee bool type for __adaptive_enabled - Declared __text: Optional[TextHandler], __tag: Optional[str], __attributes: Optional[AttributesHandler] at top of __init__ - cast(List, ...) for all XPath() call results (_find_all_elements, _find_all_elements_with_spaces) - Added Dict[float, List[Any]] for score_table, Dict[str, Any] for attributes - Changed score, checks = 0, 0 → score: float = 0; checks: int = 0 (two locations) - Renamed target → target_element in save() to avoid variable redefinition with different types - Wrapped node_text.clean() / .lower() in TextHandler(...) to preserve type `scrapling/engines/_browsers/_page.py` - Added PageInfo[SyncPage] | PageInfo[AsyncPage] union type annotation to page_info variable `scrapling/engines/_browsers/_validators.py` - Convert method_kwargs (TypedDict) to plain Dict[str, Any] before dynamic key access `scrapling/engines/_browsers/_base.py` - Added _config declaration to BaseSessionMixin - Used cast(StealthConfig, self._config) in __generate_stealth_options to access stealth-only attributes - Added Tuple[str, ...] annotation for flags - Removed redundant narrower StealthConfig type annotation on self._config in StealthySessionMixin.__validate__ - Widened SyncSession and AsyncSession fields (playwright, context, browser) to Any to support both playwright and patchright types - Added -> None to both start() methods `scrapling/engines/_browsers/_stealth.py` - Added Optional, ProxyType imports - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops - Annotated outer_box: Any at first declaration, removed duplicate type annotations in subsequent branches - Added -> None to sync and async start() - Added config: Any parameter type to _initialize_context - Removed redundant self.context: AsyncBrowserContext re-annotations in conditional branches `scrapling/engines/_browsers/_controllers.py` - Added Optional, ProxyType imports - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops - Added -> None to async start() - Removed redundant self.context: AsyncBrowserContext re-annotations `scrapling/spiders/request.py` - Added Optional import, typed _fp: Optional[bytes] = None - Removed redundant body: bytes re-annotation `scrapling/spiders/session.py` - Used separate client variable instead of reassigning session = session._client (avoids type incompatibility and fixes a bug where session._make_request was called instead of client._make_request) - Added -> None to SessionManager.__init__ `scrapling/engines/toolbelt/convertor.py` - Added list[Response] annotation for history in both sync/async methods `scrapling/engines/static.py` - FetcherClient.__init__ and AsyncFetcherClient.__init__: Added **kwargs: Any and -> None `scrapling/core/shell.py` - Wrapped re_sub(...) result in TextHandler(...) to maintain correct type - Added -> None to CurlParser.__init__ - Added full type signature to create_wrapper, replaced wrapper.__signature__ = ... with setattr(wrapper, "__signature__", ...) to satisfy mypy - Added Callable to imports
This commit is contained in:
@@ -5,16 +5,12 @@ from contextlib import contextmanager, asynccontextmanager
|
||||
from playwright.sync_api._generated import Page
|
||||
from playwright.sync_api import (
|
||||
Frame,
|
||||
Browser,
|
||||
BrowserContext,
|
||||
Playwright,
|
||||
Response as SyncPlaywrightResponse,
|
||||
)
|
||||
from playwright.async_api._generated import Page as AsyncPage
|
||||
from playwright.async_api import (
|
||||
Frame as AsyncFrame,
|
||||
Browser as AsyncBrowser,
|
||||
Playwright as AsyncPlaywright,
|
||||
Response as AsyncPlaywrightResponse,
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
)
|
||||
@@ -37,6 +33,7 @@ from scrapling.core._types import (
|
||||
Optional,
|
||||
Callable,
|
||||
TYPE_CHECKING,
|
||||
cast,
|
||||
overload,
|
||||
Tuple,
|
||||
ProxyType,
|
||||
@@ -61,12 +58,12 @@ class SyncSession:
|
||||
self.max_pages = max_pages
|
||||
self.page_pool = PagePool(max_pages)
|
||||
self._max_wait_for_page = 60
|
||||
self.playwright: Playwright | Any = None
|
||||
self.context: BrowserContext | Any = None
|
||||
self.browser: Optional[Browser] = None
|
||||
self.playwright: Any = None
|
||||
self.context: Any = None
|
||||
self.browser: Any = None
|
||||
self._is_alive = False
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
pass
|
||||
|
||||
def close(self): # pragma: no cover
|
||||
@@ -215,13 +212,13 @@ class AsyncSession:
|
||||
self.max_pages = max_pages
|
||||
self.page_pool = PagePool(max_pages)
|
||||
self._max_wait_for_page = 60
|
||||
self.playwright: AsyncPlaywright | Any = None
|
||||
self.context: AsyncBrowserContext | Any = None
|
||||
self.browser: Optional[AsyncBrowser] = None
|
||||
self.playwright: Any = None
|
||||
self.context: Any = None
|
||||
self.browser: Any = None
|
||||
self._is_alive = False
|
||||
self._lock = Lock()
|
||||
|
||||
async def start(self):
|
||||
async def start(self) -> None:
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
@@ -378,6 +375,8 @@ class AsyncSession:
|
||||
|
||||
|
||||
class BaseSessionMixin:
|
||||
_config: "PlaywrightConfig | StealthConfig"
|
||||
|
||||
@overload
|
||||
def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...
|
||||
|
||||
@@ -404,7 +403,7 @@ class BaseSessionMixin:
|
||||
return config
|
||||
|
||||
def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
|
||||
config: PlaywrightConfig | StealthConfig = self._config # type: ignore[has-type]
|
||||
config: PlaywrightConfig | StealthConfig = self._config
|
||||
self._context_options.update(
|
||||
{
|
||||
"proxy": config.proxy,
|
||||
@@ -466,7 +465,7 @@ class DynamicSessionMixin(BaseSessionMixin):
|
||||
|
||||
class StealthySessionMixin(BaseSessionMixin):
|
||||
def __validate__(self, **params):
|
||||
self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig)
|
||||
self._config = self.__validate_routine__(params, model=StealthConfig)
|
||||
self._context_options.update(
|
||||
{
|
||||
"is_mobile": False,
|
||||
@@ -482,22 +481,23 @@ class StealthySessionMixin(BaseSessionMixin):
|
||||
self.__generate_stealth_options()
|
||||
|
||||
def __generate_stealth_options(self) -> None:
|
||||
flags = tuple()
|
||||
if not self._config.cdp_url:
|
||||
config = cast(StealthConfig, self._config)
|
||||
flags: Tuple[str, ...] = tuple()
|
||||
if not config.cdp_url:
|
||||
flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS
|
||||
|
||||
if self._config.block_webrtc:
|
||||
if config.block_webrtc:
|
||||
flags += (
|
||||
"--webrtc-ip-handling-policy=disable_non_proxied_udp",
|
||||
"--force-webrtc-ip-handling-policy", # Ensures the policy is enforced
|
||||
)
|
||||
if not self._config.allow_webgl:
|
||||
if not config.allow_webgl:
|
||||
flags += (
|
||||
"--disable-webgl",
|
||||
"--disable-webgl-image-chromium",
|
||||
"--disable-webgl2",
|
||||
)
|
||||
if self._config.hide_canvas:
|
||||
if config.hide_canvas:
|
||||
flags += ("--fingerprinting-canvas-image-data-noise",)
|
||||
|
||||
super(StealthySessionMixin, self).__generate_options__(flags)
|
||||
|
||||
Reference in New Issue
Block a user