Karim shoair
5ec929435b
**Resolved all 65 mypy errors across 14 files and added type annotations to all previously untyped function bodies. Final result: 0 errors with --check-untyped-defs enabled, all 454 tests pass.** `scrapling/core/_types.py` - Removed broken Self = object fallback — now requires typing_extensions for Python < 3.11 `scrapling/core/storage.py` - Fixed str/bytes mismatch in _get_hash() — used separate _identifier_bytes variable instead of reassigning from str to bytes `scrapling/core/custom_types.py` - split() return type: Union[List, "TextHandlers"] → list[Any] (avoids LSP violation with parent list[str]) - format() kwargs: **kwargs: str → **kwargs: object (matches parent str.format signature) - AttributesHandler.__init__: Added mapping: Any = None, **kwargs: Any and -> None - json_string property: Added -> bytes return type `scrapling/core/mixins.py` - Changed self: "Selector" to self: Any on all mixin methods (mypy can't handle forward-reference self types on non-subclass mixins) - Added Dict[str, int] annotation for counter variable - Removed unused TYPE_CHECKING / Selector imports `scrapling/parser.py (~30 errors)` - Added body: str | bytes pre-annotation for dual-type if/elif assignment - Used Dict[str, Any] kwargs dict for HTMLParser(...) to bypass incomplete lxml stubs missing default_doctype - Changed base_url=url or None → base_url=url or "" (avoids str | None vs str | bytes) - bool(adaptive) to guarantee bool type for __adaptive_enabled - Declared __text: Optional[TextHandler], __tag: Optional[str], __attributes: Optional[AttributesHandler] at top of __init__ - cast(List, ...) for all XPath() call results (_find_all_elements, _find_all_elements_with_spaces) - Added Dict[float, List[Any]] for score_table, Dict[str, Any] for attributes - Changed score, checks = 0, 0 → score: float = 0; checks: int = 0 (two locations) - Renamed target → target_element in save() to avoid variable redefinition with different types - Wrapped node_text.clean() / .lower() in TextHandler(...) to preserve type `scrapling/engines/_browsers/_page.py` - Added PageInfo[SyncPage] | PageInfo[AsyncPage] union type annotation to page_info variable `scrapling/engines/_browsers/_validators.py` - Convert method_kwargs (TypedDict) to plain Dict[str, Any] before dynamic key access `scrapling/engines/_browsers/_base.py` - Added _config declaration to BaseSessionMixin - Used cast(StealthConfig, self._config) in __generate_stealth_options to access stealth-only attributes - Added Tuple[str, ...] annotation for flags - Removed redundant narrower StealthConfig type annotation on self._config in StealthySessionMixin.__validate__ - Widened SyncSession and AsyncSession fields (playwright, context, browser) to Any to support both playwright and patchright types - Added -> None to both start() methods `scrapling/engines/_browsers/_stealth.py` - Added Optional, ProxyType imports - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops - Annotated outer_box: Any at first declaration, removed duplicate type annotations in subsequent branches - Added -> None to sync and async start() - Added config: Any parameter type to _initialize_context - Removed redundant self.context: AsyncBrowserContext re-annotations in conditional branches `scrapling/engines/_browsers/_controllers.py` - Added Optional, ProxyType imports - Annotated proxy: Optional[ProxyType] in both sync/async fetch loops - Added -> None to async start() - Removed redundant self.context: AsyncBrowserContext re-annotations `scrapling/spiders/request.py` - Added Optional import, typed _fp: Optional[bytes] = None - Removed redundant body: bytes re-annotation `scrapling/spiders/session.py` - Used separate client variable instead of reassigning session = session._client (avoids type incompatibility and fixes a bug where session._make_request was called instead of client._make_request) - Added -> None to SessionManager.__init__ `scrapling/engines/toolbelt/convertor.py` - Added list[Response] annotation for history in both sync/async methods `scrapling/engines/static.py` - FetcherClient.__init__ and AsyncFetcherClient.__init__: Added **kwargs: Any and -> None `scrapling/core/shell.py` - Wrapped re_sub(...) result in TextHandler(...) to maintain correct type - Added -> None to CurlParser.__init__ - Added full type signature to create_wrapper, replaced wrapper.__signature__ = ... with setattr(wrapper, "__signature__", ...) to satisfy mypy - Added Callable to imports
86 lines
3.5 KiB
Python
86 lines
3.5 KiB
Python
from scrapling.core._types import Any, Dict
|
|
|
|
|
|
class SelectorsGeneration:
|
|
"""
|
|
Functions for generating selectors
|
|
Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
|
|
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
|
|
"""
|
|
|
|
# Note: This is a mixin class meant to be used with Selector.
|
|
# The methods access Selector attributes (._root, .parent, .attrib, .tag, etc.)
|
|
# through self, which will be a Selector instance at runtime.
|
|
|
|
def _general_selection(self: Any, selection: str = "css", full_path: bool = False) -> str:
|
|
"""Generate a selector for the current element.
|
|
:return: A string of the generated selector.
|
|
"""
|
|
if self._is_text_node(self._root):
|
|
return ""
|
|
|
|
selectorPath = []
|
|
target = self
|
|
css = selection.lower() == "css"
|
|
while target is not None:
|
|
if target.parent:
|
|
if target.attrib.get("id"):
|
|
# id is enough
|
|
part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']"
|
|
selectorPath.append(part)
|
|
if not full_path:
|
|
return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath))
|
|
else:
|
|
part = f"{target.tag}"
|
|
# We won't use classes anymore because I some websites share exact classes between elements
|
|
# classes = target.attrib.get('class', '').split()
|
|
# if classes and css:
|
|
# part += f".{'.'.join(classes)}"
|
|
# else:
|
|
counter: Dict[str, int] = {}
|
|
for child in target.parent.children:
|
|
counter.setdefault(child.tag, 0)
|
|
counter[child.tag] += 1
|
|
if child._root == target._root:
|
|
break
|
|
|
|
if counter[target.tag] > 1:
|
|
part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]"
|
|
|
|
selectorPath.append(part)
|
|
target = target.parent
|
|
if target is None or target.tag == "html":
|
|
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
|
|
else:
|
|
break
|
|
|
|
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
|
|
|
|
@property
|
|
def generate_css_selector(self: Any) -> str:
|
|
"""Generate a CSS selector for the current element
|
|
:return: A string of the generated selector.
|
|
"""
|
|
return self._general_selection()
|
|
|
|
@property
|
|
def generate_full_css_selector(self: Any) -> str:
|
|
"""Generate a complete CSS selector for the current element
|
|
:return: A string of the generated selector.
|
|
"""
|
|
return self._general_selection(full_path=True)
|
|
|
|
@property
|
|
def generate_xpath_selector(self: Any) -> str:
|
|
"""Generate an XPath selector for the current element
|
|
:return: A string of the generated selector.
|
|
"""
|
|
return self._general_selection("xpath")
|
|
|
|
@property
|
|
def generate_full_xpath_selector(self: Any) -> str:
|
|
"""Generate a complete XPath selector for the current element
|
|
:return: A string of the generated selector.
|
|
"""
|
|
return self._general_selection("xpath", full_path=True)
|