diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index f19e706..f2fc097 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -32,6 +32,7 @@ from typing import ( Coroutine, SupportsIndex, ) +from typing_extensions import Self, Unpack # Proxy can be a string URL or a dict (Playwright format: {"server": "...", "username": "...", "password": "..."}) ProxyType = Union[str, Dict[str, str]] @@ -41,27 +42,6 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"] extraction_types = Literal["text", "html", "markdown"] StrOrBytes = Union[str, bytes] -if TYPE_CHECKING: # pragma: no cover - from typing_extensions import Unpack -else: # pragma: no cover - - class _Unpack: - @staticmethod - def __getitem__(*args, **kwargs): - pass - - Unpack = _Unpack() - - -try: - # Python 3.11+ - from typing import Self # novermin -except ImportError: # pragma: no cover - try: - from typing_extensions import Self # Backport - except ImportError: - Self = object - # Copied from `playwright._impl._api_structures.SetCookieParam` class SetCookieParam(TypedDict, total=False): diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 45ab4d5..d06ac29 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -35,9 +35,7 @@ class TextHandler(str): lst = super().__getitem__(key) return TextHandler(lst) - def split( - self, sep: str | None = None, maxsplit: SupportsIndex = -1 - ) -> Union[List, "TextHandlers"]: # pragma: no cover + def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[Any]: # pragma: no cover return TextHandlers([TextHandler(s) for s in super().split(sep, maxsplit)]) def strip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover @@ -61,7 +59,7 @@ class TextHandler(str): def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().expandtabs(tabsize)) - def format(self, *args: object, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover + def format(self, *args: object, **kwargs: object) -> Union[str, "TextHandler"]: # pragma: no cover return TextHandler(super().format(*args, **kwargs)) def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover @@ -291,7 +289,7 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): __slots__ = ("_data",) - def __init__(self, mapping=None, **kwargs): + def __init__(self, mapping: Any = None, **kwargs: Any) -> None: mapping = ( {key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()} if mapping is not None @@ -324,8 +322,8 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): yield AttributesHandler({key: value}) @property - def json_string(self): - """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error""" + def json_string(self) -> bytes: + """Convert current attributes to JSON bytes if the attributes are JSON serializable otherwise throws error""" return dumps(dict(self._data)) def __getitem__(self, key: str) -> _TextHandlerType: diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py index 0b66c07..c2e7420 100644 --- a/scrapling/core/mixins.py +++ b/scrapling/core/mixins.py @@ -1,7 +1,4 @@ -from scrapling.core._types import TYPE_CHECKING - -if TYPE_CHECKING: - from scrapling.parser import Selector +from scrapling.core._types import Any, Dict class SelectorsGeneration: @@ -11,7 +8,11 @@ class SelectorsGeneration: Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591 """ - def _general_selection(self: "Selector", selection: str = "css", full_path: bool = False) -> str: # type: ignore[name-defined] + # Note: This is a mixin class meant to be used with Selector. + # The methods access Selector attributes (._root, .parent, .attrib, .tag, etc.) + # through self, which will be a Selector instance at runtime. + + def _general_selection(self: Any, selection: str = "css", full_path: bool = False) -> str: """Generate a selector for the current element. :return: A string of the generated selector. """ @@ -36,7 +37,7 @@ class SelectorsGeneration: # if classes and css: # part += f".{'.'.join(classes)}" # else: - counter = {} + counter: Dict[str, int] = {} for child in target.parent.children: counter.setdefault(child.tag, 0) counter[child.tag] += 1 @@ -56,28 +57,28 @@ class SelectorsGeneration: return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath)) @property - def generate_css_selector(self: "Selector") -> str: # type: ignore[name-defined] + def generate_css_selector(self: Any) -> str: """Generate a CSS selector for the current element :return: A string of the generated selector. """ return self._general_selection() @property - def generate_full_css_selector(self: "Selector") -> str: # type: ignore[name-defined] + def generate_full_css_selector(self: Any) -> str: """Generate a complete CSS selector for the current element :return: A string of the generated selector. """ return self._general_selection(full_path=True) @property - def generate_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined] + def generate_xpath_selector(self: Any) -> str: """Generate an XPath selector for the current element :return: A string of the generated selector. """ return self._general_selection("xpath") @property - def generate_full_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined] + def generate_full_xpath_selector(self: Any) -> str: """Generate a complete XPath selector for the current element :return: A string of the generated selector. """ diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 3a1511e..0bd6efc 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -30,6 +30,7 @@ from scrapling.core.custom_types import TextHandler from scrapling.engines.toolbelt.custom import Response from scrapling.core.utils._shell import _ParseHeaders, _CookieParser from scrapling.core._types import ( + Callable, Dict, Any, cast, @@ -82,7 +83,7 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no cover class CurlParser: """Builds the argument parser for relevant curl flags from DevTools.""" - def __init__(self): + def __init__(self) -> None: from scrapling.fetchers import Fetcher as __Fetcher self.__fetcher = __Fetcher @@ -467,19 +468,21 @@ Type 'exit' or press Ctrl+D to exit. return result - def create_wrapper(self, func, get_signature=True, signature_name=None): + def create_wrapper( + self, func: Callable, get_signature: bool = True, signature_name: Optional[str] = None + ) -> Callable: """Create a wrapper that preserves function signature but updates page""" @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: result = func(*args, **kwargs) return self.update_page(result) if get_signature: # Explicitly preserve and unpack signature for IPython introspection and autocompletion - wrapper.__signature__ = _unpack_signature(func, signature_name) # pyright: ignore + setattr(wrapper, "__signature__", _unpack_signature(func, signature_name)) else: - wrapper.__signature__ = signature(func) # pyright: ignore + setattr(wrapper, "__signature__", signature(func)) return wrapper @@ -601,7 +604,7 @@ class Convertor: " ", ): # Remove consecutive white-spaces - txt_content = re_sub(f"[{s}]+", s, txt_content) + txt_content = TextHandler(re_sub(f"[{s}]+", s, txt_content)) yield txt_content yield "" diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index 8e04599..d93a320 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -63,12 +63,11 @@ class StorageSystemMixin(ABC): # pragma: no cover def _get_hash(identifier: str) -> str: """If you want to hash identifier in your storage system, use this safer""" _identifier = identifier.lower().strip() - if isinstance(_identifier, str): - # Hash functions have to take bytes - _identifier = _identifier.encode("utf-8") + # Hash functions have to take bytes + _identifier_bytes = _identifier.encode("utf-8") - hash_value = sha256(_identifier).hexdigest() - return f"{hash_value}_{len(_identifier)}" # Length to reduce collision chance + hash_value = sha256(_identifier_bytes).hexdigest() + return f"{hash_value}_{len(_identifier_bytes)}" # Length to reduce collision chance @lru_cache(1, typed=True) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 8bd442d..e379882 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -5,16 +5,12 @@ from contextlib import contextmanager, asynccontextmanager from playwright.sync_api._generated import Page from playwright.sync_api import ( Frame, - Browser, BrowserContext, - Playwright, Response as SyncPlaywrightResponse, ) from playwright.async_api._generated import Page as AsyncPage from playwright.async_api import ( Frame as AsyncFrame, - Browser as AsyncBrowser, - Playwright as AsyncPlaywright, Response as AsyncPlaywrightResponse, BrowserContext as AsyncBrowserContext, ) @@ -37,6 +33,7 @@ from scrapling.core._types import ( Optional, Callable, TYPE_CHECKING, + cast, overload, Tuple, ProxyType, @@ -61,12 +58,12 @@ class SyncSession: self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 - self.playwright: Playwright | Any = None - self.context: BrowserContext | Any = None - self.browser: Optional[Browser] = None + self.playwright: Any = None + self.context: Any = None + self.browser: Any = None self._is_alive = False - def start(self): + def start(self) -> None: pass def close(self): # pragma: no cover @@ -215,13 +212,13 @@ class AsyncSession: self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 - self.playwright: AsyncPlaywright | Any = None - self.context: AsyncBrowserContext | Any = None - self.browser: Optional[AsyncBrowser] = None + self.playwright: Any = None + self.context: Any = None + self.browser: Any = None self._is_alive = False self._lock = Lock() - async def start(self): + async def start(self) -> None: pass async def close(self): @@ -378,6 +375,8 @@ class AsyncSession: class BaseSessionMixin: + _config: "PlaywrightConfig | StealthConfig" + @overload def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ... @@ -404,7 +403,7 @@ class BaseSessionMixin: return config def __generate_options__(self, extra_flags: Tuple | None = None) -> None: - config: PlaywrightConfig | StealthConfig = self._config # type: ignore[has-type] + config: PlaywrightConfig | StealthConfig = self._config self._context_options.update( { "proxy": config.proxy, @@ -466,7 +465,7 @@ class DynamicSessionMixin(BaseSessionMixin): class StealthySessionMixin(BaseSessionMixin): def __validate__(self, **params): - self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig) + self._config = self.__validate_routine__(params, model=StealthConfig) self._context_options.update( { "is_mobile": False, @@ -482,22 +481,23 @@ class StealthySessionMixin(BaseSessionMixin): self.__generate_stealth_options() def __generate_stealth_options(self) -> None: - flags = tuple() - if not self._config.cdp_url: + config = cast(StealthConfig, self._config) + flags: Tuple[str, ...] = tuple() + if not config.cdp_url: flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS - if self._config.block_webrtc: + if config.block_webrtc: flags += ( "--webrtc-ip-handling-policy=disable_non_proxied_udp", "--force-webrtc-ip-handling-policy", # Ensures the policy is enforced ) - if not self._config.allow_webgl: + if not config.allow_webgl: flags += ( "--disable-webgl", "--disable-webgl-image-chromium", "--disable-webgl2", ) - if self._config.hide_canvas: + if config.hide_canvas: flags += ("--fingerprinting-canvas-image-data-noise",) super(StealthySessionMixin, self).__generate_options__(flags) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 5f0b247..5d9e801 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -8,11 +8,10 @@ from playwright.sync_api import ( from playwright.async_api import ( async_playwright, Locator as AsyncLocator, - BrowserContext as AsyncBrowserContext, ) from scrapling.core.utils import log -from scrapling.core._types import Unpack +from scrapling.core._types import Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer @@ -134,6 +133,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin): ) for attempt in range(self._config.retries): + proxy: Optional[ProxyType] = None if self._config.proxy_rotator and static_proxy is None: proxy = self._config.proxy_rotator.get_proxy() else: @@ -238,7 +238,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): self.__validate__(**kwargs) super().__init__(max_pages=self._config.max_pages) - async def start(self): + async def start(self) -> None: """Create a browser for this instance and context.""" if not self.playwright: self.playwright = await async_playwright().start() @@ -246,16 +246,14 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): if self._config.cdp_url: self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) if not self._config.proxy_rotator and self.browser: - self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options) + self.context = await self.browser.new_context(**self._context_options) elif self._config.proxy_rotator: self.browser = await self.playwright.chromium.launch(**self._browser_options) else: persistent_options = ( self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir} ) - self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - **persistent_options - ) + self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options) if self.context: self.context = await self._initialize_context(self._config, self.context) @@ -304,6 +302,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): ) for attempt in range(self._config.retries): + proxy: Optional[ProxyType] = None if self._config.proxy_rotator and static_proxy is None: proxy = self._config.proxy_rotator.get_proxy() else: diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 655d3d1..481016e 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -61,7 +61,9 @@ class PagePool: raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached") if isinstance(page, AsyncPage): - page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", "")) + page_info: PageInfo[SyncPage] | PageInfo[AsyncPage] = cast( + PageInfo[AsyncPage], PageInfo(page, "ready", "") + ) else: page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", "")) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 730cd7e..3c9ea58 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -13,7 +13,7 @@ from patchright.sync_api import sync_playwright from patchright.async_api import async_playwright from scrapling.core.utils import log -from scrapling.core._types import Any, Unpack +from scrapling.core._types import Any, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer @@ -78,7 +78,7 @@ class StealthySession(SyncSession, StealthySessionMixin): self.__validate__(**kwargs) super().__init__() - def start(self): + def start(self) -> None: """Create a browser for this instance and context.""" if not self.playwright: self.playwright = sync_playwright().start() @@ -146,7 +146,7 @@ class StealthySession(SyncSession, StealthySessionMixin): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared page.wait_for_timeout(500) - outer_box = {} + outer_box: Any = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: self._wait_for_page_stability(iframe, True, False) @@ -156,14 +156,14 @@ class StealthySession(SyncSession, StealthySessionMixin): # Double-checking that the iframe is loaded page.wait_for_timeout(500) - outer_box: Any = iframe.frame_element().bounding_box() + outer_box = iframe.frame_element().bounding_box() if not iframe or not outer_box: if "Just a moment..." not in (ResponseFactory._get_page_content(page)): log.info("Cloudflare captcha is solved") return - outer_box: Any = page.locator(box_selector).last.bounding_box() + outer_box = page.locator(box_selector).last.bounding_box() # Calculate the Captcha coordinates for any viewport captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) @@ -223,6 +223,7 @@ class StealthySession(SyncSession, StealthySessionMixin): ) for attempt in range(self._config.retries): + proxy: Optional[ProxyType] = None if self._config.proxy_rotator and static_proxy is None: proxy = self._config.proxy_rotator.get_proxy() else: @@ -335,7 +336,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): self.__validate__(**kwargs) super().__init__(max_pages=self._config.max_pages) - async def start(self): + async def start(self) -> None: """Create a browser for this instance and context.""" if not self.playwright: self.playwright = await async_playwright().start() @@ -344,16 +345,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) if not self._config.proxy_rotator: assert self.browser is not None - self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options) + self.context = await self.browser.new_context(**self._context_options) elif self._config.proxy_rotator: self.browser = await self.playwright.chromium.launch(**self._browser_options) else: persistent_options = ( self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir} ) - self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - **persistent_options - ) + self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options) if self.context: self.context = await self._initialize_context(self._config, self.context) @@ -367,7 +366,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - async def _initialize_context(self, config, ctx: AsyncBrowserContext) -> AsyncBrowserContext: + async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext: """Initialize the browser context.""" for script in _compiled_stealth_scripts(): await ctx.add_init_script(script=script) @@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared await page.wait_for_timeout(500) - outer_box = {} + outer_box: Any = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: await self._wait_for_page_stability(iframe, True, False) @@ -414,14 +413,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): # Double-checking that the iframe is loaded await page.wait_for_timeout(500) - outer_box: Any = await (await iframe.frame_element()).bounding_box() + outer_box = await (await iframe.frame_element()).bounding_box() if not iframe or not outer_box: if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): log.info("Cloudflare captcha is solved") return - outer_box: Any = await page.locator(box_selector).last.bounding_box() + outer_box = await page.locator(box_selector).last.bounding_box() # Calculate the Captcha coordinates for any viewport captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) @@ -482,6 +481,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): ) for attempt in range(self._config.retries): + proxy: Optional[ProxyType] = None if self._config.proxy_rotator and static_proxy is None: proxy = self._config.proxy_rotator.get_proxy() else: diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index af2baef..e2424f4 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -157,15 +157,16 @@ def validate_fetch( session: Any, model: type[PlaywrightConfig] | type[StealthConfig], ) -> _fetch_params: # pragma: no cover - result = {} - overrides = {} + result: Dict[str, Any] = {} + overrides: Dict[str, Any] = {} + kwargs_dict: Dict[str, Any] = dict(method_kwargs) # Get all field names that _fetch_params needs fetch_param_fields = {f.name for f in fields(_fetch_params)} for key in fetch_param_fields: - if key in method_kwargs: - overrides[key] = method_kwargs[key] + if key in kwargs_dict: + overrides[key] = kwargs_dict[key] elif hasattr(session, "_config") and hasattr(session._config, key): result[key] = getattr(session._config, key) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 884fe0c..204c105 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -753,7 +753,7 @@ class FetcherSession: class FetcherClient(_SyncSessionLogic): __slots__ = ("__enter__", "__exit__") - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.__enter__: Any = None self.__exit__: Any = None @@ -763,7 +763,7 @@ class FetcherClient(_SyncSessionLogic): class AsyncFetcherClient(_ASyncSessionLogic): __slots__ = ("__aenter__", "__aexit__") - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.__aenter__: Any = None self.__aexit__: Any = None diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 4b4caa2..1a8d799 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -38,7 +38,7 @@ class ResponseFactory: @classmethod def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]: """Process response history to build a list of `Response` objects""" - history = [] + history: list[Response] = [] current_request = first_response.request.redirected_from try: @@ -101,6 +101,7 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. + :param meta: Additional meta data to be saved with the response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -145,7 +146,7 @@ class ResponseFactory: cls, first_response: AsyncResponse, parser_arguments: Dict ) -> list[Response]: """Process response history to build a list of `Response` objects""" - history = [] + history: list[Response] = [] current_request = first_response.request.redirected_from try: @@ -238,6 +239,7 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. + :param meta: Additional meta data to be saved with the response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response diff --git a/scrapling/parser.py b/scrapling/parser.py index 1ae6db1..820bc71 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -118,8 +118,11 @@ class Selector(SelectorsGeneration): if root is None and content is None: raise ValueError("Selector class needs HTML content, or root arguments to work") - self.__text = None + self.__text: Optional[TextHandler] = None + self.__tag: Optional[str] = None + self.__attributes: Optional[AttributesHandler] = None if root is None: + body: str | bytes if isinstance(content, str): body = content.strip().replace("\x00", "") or "" elif isinstance(content, bytes): @@ -128,17 +131,18 @@ class Selector(SelectorsGeneration): raise TypeError(f"content argument must be str or bytes, got {type(content)}") # https://lxml.de/api/lxml.etree.HTMLParser-class.html - parser = HTMLParser( + _parser_kwargs: Dict[str, Any] = dict( recover=True, remove_blank_text=True, remove_comments=(not keep_comments), encoding=encoding, compact=True, huge_tree=huge_tree, - default_doctype=True, + default_doctype=True, # Supported by lxml but missing from stubs strip_cdata=(not keep_cdata), ) - self._root = cast(HtmlElement, fromstring(body or "", parser=parser, base_url=url or None)) + parser = HTMLParser(**_parser_kwargs) + self._root = cast(HtmlElement, fromstring(body or "", parser=parser, base_url=url or "")) self._raw_body = content else: @@ -164,7 +168,7 @@ class Selector(SelectorsGeneration): self._root = cast(HtmlElement, root) self._raw_body = "" - self.__adaptive_enabled = adaptive + self.__adaptive_enabled = bool(adaptive) if self.__adaptive_enabled: if _storage is not None: @@ -277,8 +281,8 @@ class Selector(SelectorsGeneration): if self._is_text_node(self._root): return "#text" if not self.__tag: - self.__tag = self._root.tag - return self.__tag + self.__tag = str(self._root.tag) + return self.__tag or "" @property def text(self) -> TextHandler: @@ -313,11 +317,11 @@ class Selector(SelectorsGeneration): if self._is_text_node(self._root): return TextHandler(str(self._root)) - ignored_elements = set() + ignored_elements: set[Any] = set() if ignore_tags: for element in self._root.iter(*ignore_tags): ignored_elements.add(element) - ignored_elements.update(set(_find_all_elements(element))) + ignored_elements.update(cast(list, _find_all_elements(element))) _all_strings = [] for node in self._root.iter(): @@ -395,7 +399,7 @@ class Selector(SelectorsGeneration): """Return all elements under the current element in the DOM tree""" if self._is_text_node(self._root): return Selectors() - below = _find_all_elements(self._root) + below = cast(List, _find_all_elements(self._root)) return self.__elements_convertor(below) if below is not None else Selectors() @property @@ -533,7 +537,7 @@ class Selector(SelectorsGeneration): :param selector_type: If True, the return result will be converted to `Selectors` object :return: List of pure HTML elements that got the highest matching score or 'Selectors' object """ - score_table = {} + score_table: Dict[float, List[Any]] = {} # Note: `element` will most likely always be a dictionary at this point. if isinstance(element, self.__class__): element = element._root @@ -541,11 +545,11 @@ class Selector(SelectorsGeneration): if issubclass(type(element), HtmlElement): element = _StorageTools.element_to_dict(element) - for node in _find_all_elements(self._root): + for node in cast(List, _find_all_elements(self._root)): # Collect all elements in the page, then for each element get the matching score of it against the node. # Hence: the code doesn't stop even if the score was 100% # because there might be another element(s) left in page with the same score - score = self.__calculate_similarity_score(element, node) + score = self.__calculate_similarity_score(cast(Dict, element), node) score_table.setdefault(score, []).append(node) if score_table: @@ -710,7 +714,7 @@ class Selector(SelectorsGeneration): if not args and not kwargs: raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.") - attributes = dict() + attributes: Dict[str, Any] = dict() tags: Set[str] = set() patterns: Set[Pattern] = set() results, functions, selectors = Selectors(), [], [] @@ -809,21 +813,19 @@ class Selector(SelectorsGeneration): :param candidate: The element to compare with the original element. :return: A percentage score of how similar is the candidate to the original element """ - score, checks = 0, 0 + score: float = 0 + checks: int = 0 data = _StorageTools.element_to_dict(candidate) - # Possible TODO: - # Study the idea of giving weight to each test below so some are more important than others - # Current results: With weights some websites had better score while it was worse for others - score += 1 if original["tag"] == data["tag"] else 0 # * 0.3 # 30% + score += 1 if original["tag"] == data["tag"] else 0 checks += 1 if original["text"]: - score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio() # * 0.3 # 30% + score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio() checks += 1 # if both don't have attributes, it still counts for something! - score += self.__calculate_dict_diff(original["attributes"], data["attributes"]) # * 0.3 # 30% + score += self.__calculate_dict_diff(original["attributes"], data["attributes"]) checks += 1 # Separate similarity test for class, id, href,... this will help in full structural changes @@ -838,23 +840,19 @@ class Selector(SelectorsGeneration): None, original["attributes"][attrib], data["attributes"].get(attrib) or "", - ).ratio() # * 0.3 # 30% + ).ratio() checks += 1 - score += SequenceMatcher(None, original["path"], data["path"]).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["path"], data["path"]).ratio() checks += 1 if original.get("parent_name"): # Then we start comparing parents' data if data.get("parent_name"): - score += SequenceMatcher( - None, original["parent_name"], data.get("parent_name") or "" - ).ratio() # * 0.2 # 20% + score += SequenceMatcher(None, original["parent_name"], data.get("parent_name") or "").ratio() checks += 1 - score += self.__calculate_dict_diff( - original["parent_attribs"], data.get("parent_attribs") or {} - ) # * 0.2 # 20% + score += self.__calculate_dict_diff(original["parent_attribs"], data.get("parent_attribs") or {}) checks += 1 if original["parent_text"]: @@ -862,14 +860,14 @@ class Selector(SelectorsGeneration): None, original["parent_text"], data.get("parent_text") or "", - ).ratio() # * 0.1 # 10% + ).ratio() checks += 1 # else: # # The original element has a parent and this one not, this is not a good sign # score -= 0.1 if original.get("siblings"): - score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio() # * 0.1 # 10% + score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio() checks += 1 # How % sure? let's see @@ -890,14 +888,14 @@ class Selector(SelectorsGeneration): the docs for more info. """ if self.__adaptive_enabled: - target = element - if isinstance(target, self.__class__): - target: HtmlElement = target._root + target_element: Any = element + if isinstance(target_element, self.__class__): + target_element = target_element._root - if self._is_text_node(target): - target: HtmlElement = target.getparent() + if self._is_text_node(target_element): + target_element = target_element.getparent() - self._storage.save(target, identifier) + self._storage.save(target_element, identifier) else: raise RuntimeError( "Can't use `adaptive` features while it's disabled globally, you have to start a new class instance." @@ -987,7 +985,8 @@ class Selector(SelectorsGeneration): candidate_attributes = ( self.__get_attributes(candidate, ignore_attributes) if ignore_attributes else candidate.attrib ) - score, checks = 0, 0 + score: float = 0 + checks: int = 0 if original_attributes: score += sum( @@ -1116,16 +1115,16 @@ class Selector(SelectorsGeneration): if not case_sensitive: text = text.lower() - possible_targets = _find_all_elements_with_spaces(self._root) + possible_targets = cast(List, _find_all_elements_with_spaces(self._root)) if possible_targets: for node in self.__elements_convertor(possible_targets): """Check if element matches given text otherwise, traverse the children tree and iterate""" - node_text = node.text + node_text: TextHandler = node.text if clean_match: - node_text = node_text.clean() + node_text = TextHandler(node_text.clean()) if not case_sensitive: - node_text = node_text.lower() + node_text = TextHandler(node_text.lower()) if partial: if text in node_text: @@ -1178,7 +1177,7 @@ class Selector(SelectorsGeneration): results = Selectors() - possible_targets = _find_all_elements_with_spaces(self._root) + possible_targets = cast(List, _find_all_elements_with_spaces(self._root)) if possible_targets: for node in self.__elements_convertor(possible_targets): """Check if element matches given regex otherwise, traverse the children tree and iterate""" diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py index 3351100..edfc387 100644 --- a/scrapling/spiders/request.py +++ b/scrapling/spiders/request.py @@ -7,7 +7,7 @@ import orjson from w3lib.url import canonicalize_url from scrapling.engines.toolbelt.custom import Response -from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Union, Tuple, TYPE_CHECKING +from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Optional, Union, Tuple, TYPE_CHECKING if TYPE_CHECKING: from scrapling.spiders.spider import Spider @@ -42,7 +42,7 @@ class Request: self.meta: dict[str, Any] = meta if meta else {} self._retry_count: int = _retry_count self._session_kwargs = kwargs if kwargs else {} - self._fp = None + self._fp: Optional[bytes] = None def copy(self) -> "Request": """Create a copy of this request.""" @@ -89,7 +89,7 @@ class Request: body = b"" else: post_data = self._session_kwargs.get("json", {}) - body: bytes = orjson.dumps(post_data) if post_data else b"" + body = orjson.dumps(post_data) if post_data else b"" data: Dict[str, str | Tuple] = { "sid": self.sid, diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py index 8f5e7af..cc07042 100644 --- a/scrapling/spiders/session.py +++ b/scrapling/spiders/session.py @@ -12,7 +12,7 @@ Session = FetcherSession | AsyncDynamicSession | AsyncStealthySession class SessionManager: """Manages pre-configured session instances.""" - def __init__(self): + def __init__(self) -> None: self._sessions: dict[str, Session] = {} self._default_session_id: str | None = None self._started: bool = False @@ -109,17 +109,17 @@ class SessionManager: await session.__aenter__() if isinstance(session, FetcherSession): - session = session._client + client = session._client - if isinstance(session, _ASyncSessionLogic): - response = await session._make_request( + if isinstance(client, _ASyncSessionLogic): + response = await client._make_request( method=cast(SUPPORTED_HTTP_METHODS, request._session_kwargs.pop("method", "GET")), url=request.url, **request._session_kwargs, ) else: # Sync session or other types - shouldn't happen in async context - raise TypeError(f"Session type {type(session)} not supported for async fetch") + raise TypeError(f"Session type {type(client)} not supported for async fetch") else: response = await session.fetch(url=request.url, **request._session_kwargs)