freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Making all the codebase acceptable by PyRight

Browse Source 
Also fixes #97
This commit is contained in:
Karim shoair
2025-10-05 04:03:39 +03:00
parent e149c715dd
commit debe03256b
21 changed files with 306 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_types.py
+3
View File
@@ -12,9 +12,11 @@ from typing import (
Generator,
Iterable,
List,
Set,
Literal,
Optional,
Pattern,
Sequence,
Tuple,
TypeVar,
Union,
@@ -22,6 +24,7 @@ from typing import (
Mapping,
Awaitable,
Protocol,
Coroutine,
SupportsIndex,
)
scrapling/core/ai.py
+2 -1
View File
@@ -20,6 +20,7 @@ from scrapling.core._types import (
Mapping,
Dict,
List,
Any,
SelectorWaitStates,
Generator,
)
@@ -171,7 +172,7 @@ class ScraplingMCPServer:
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
"""
async with FetcherSession() as session:
tasks = [
tasks: List[Any] = [
session.get(
url,
auth=auth,
scrapling/core/custom_types.py
+20 -27
View File
@@ -5,6 +5,7 @@ from re import compile as re_compile, UNICODE, IGNORECASE
from orjson import dumps, loads
from scrapling.core._types import (
Any,
cast,
Dict,
List,
@@ -14,7 +15,6 @@ from scrapling.core._types import (
Literal,
Pattern,
Iterable,
Optional,
Generator,
SupportsIndex,
)
@@ -33,23 +33,20 @@ class TextHandler(str):
def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": # pragma: no cover
lst = super().__getitem__(key)
return cast(_TextHandlerType, TextHandler(lst))
return TextHandler(lst)
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": # pragma: no cover
return TextHandlers(
cast(
List[_TextHandlerType],
[TextHandler(s) for s in super().split(sep, maxsplit)],
)
)
def split(
self, sep: str | None = None, maxsplit: SupportsIndex = -1
) -> Union[List, "TextHandlers"]: # pragma: no cover
return TextHandlers([TextHandler(s) for s in super().split(sep, maxsplit)])
def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
def strip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
def lstrip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
def rstrip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -64,7 +61,7 @@ class TextHandler(str):
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover
def format(self, *args: object, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -131,10 +128,11 @@ class TextHandler(str):
def re(
self,
regex: str | Pattern,
check_match: Literal[True],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
*,
check_match: Literal[True],
) -> bool: ...
@overload
@@ -179,19 +177,14 @@ class TextHandler(str):
results = flatten(results)
if not replace_entities:
return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
return TextHandlers([TextHandler(string) for string in results])
return TextHandlers(
cast(
List[_TextHandlerType],
[TextHandler(_replace_entities(s)) for s in results],
)
)
return TextHandlers([TextHandler(_replace_entities(s)) for s in results])
def re_first(
self,
regex: str | Pattern,
default=None,
default: Any = None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
@@ -232,8 +225,8 @@ class TextHandlers(List[TextHandler]):
def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]:
lst = super().__getitem__(pos)
if isinstance(pos, slice):
return TextHandlers(cast(List[_TextHandlerType], lst))
return cast(_TextHandlerType, TextHandler(lst))
return TextHandlers(cast(List[TextHandler], lst))
return TextHandler(cast(TextHandler, lst))
def re(
self,
@@ -256,7 +249,7 @@ class TextHandlers(List[TextHandler]):
def re_first(
self,
regex: str | Pattern,
default=None,
default: Any = None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
@@ -309,9 +302,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
)
# Fastest read-only mapping type
self._data = MappingProxyType(mapping)
self._data: Mapping[str, Any] = MappingProxyType(mapping)
def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]:
def get(self, key: str, default: Any = None) -> _TextHandlerType:
"""Acts like the standard dictionary `.get()` method"""
return self._data.get(key, default)
scrapling/core/mixins.py
+15 -9
View File
@@ -1,3 +1,9 @@
from scrapling.core._types import TYPE_CHECKING
if TYPE_CHECKING:
from scrapling.parser import Selector
class SelectorsGeneration:
"""
Functions for generating selectors
@@ -5,7 +11,7 @@ class SelectorsGeneration:
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
"""
def __general_selection(self, selection: str = "css", full_path: bool = False) -> str:
def _general_selection(self: "Selector", selection: str = "css", full_path: bool = False) -> str: # type: ignore[name-defined]
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
@@ -47,29 +53,29 @@ class SelectorsGeneration:
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
@property
def generate_css_selector(self) -> str:
def generate_css_selector(self: "Selector") -> str: # type: ignore[name-defined]
"""Generate a CSS selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection()
return self._general_selection()
@property
def generate_full_css_selector(self) -> str:
def generate_full_css_selector(self: "Selector") -> str: # type: ignore[name-defined]
"""Generate a complete CSS selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection(full_path=True)
return self._general_selection(full_path=True)
@property
def generate_xpath_selector(self) -> str:
def generate_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]
"""Generate an XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection("xpath")
return self._general_selection("xpath")
@property
def generate_full_xpath_selector(self) -> str:
def generate_full_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]
"""Generate a complete XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection("xpath", full_path=True)
return self._general_selection("xpath", full_path=True)
scrapling/core/shell.py
+4 -3
View File
@@ -31,6 +31,7 @@ from scrapling.core._types import (
Optional,
Dict,
Any,
cast,
extraction_types,
Generator,
)
@@ -540,15 +541,15 @@ class Convertor:
raise ValueError(f"Unknown extraction type: {extraction_type}")
else:
if main_content_only:
page = page.css_first("body") or page
page = cast(Selector, page.css_first("body")) or page
pages = [page] if not css_selector else page.css(css_selector)
pages = [page] if not css_selector else cast(Selectors, page.css(css_selector))
for page in pages:
match extraction_type:
case "markdown":
yield cls._convert_to_markdown(page.html_content)
case "html":
yield page.body
yield page.html_content
case "text":
txt_content = page.get_all_text(strip=True)
for s in (
scrapling/core/storage.py
+5 -5
View File
@@ -56,13 +56,13 @@ class StorageSystemMixin(ABC): # pragma: no cover
@lru_cache(128, typed=True)
def _get_hash(identifier: str) -> str:
"""If you want to hash identifier in your storage system, use this safer"""
identifier = identifier.lower().strip()
if isinstance(identifier, str):
_identifier = identifier.lower().strip()
if isinstance(_identifier, str):
# Hash functions have to take bytes
identifier = identifier.encode("utf-8")
_identifier = _identifier.encode("utf-8")
hash_value = sha256(identifier).hexdigest()
return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance
hash_value = sha256(_identifier).hexdigest()
return f"{hash_value}_{len(_identifier)}" # Length to reduce collision chance
@lru_cache(1, typed=True)
scrapling/core/translator.py
+13 -8
View File
@@ -10,24 +10,23 @@ So you don't have to learn a new selectors/api method like what bs4 done with so
from functools import lru_cache
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.xpath import ExpressionError, XPathExpr as OriginalXPathExpr
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
from scrapling.core._types import Any, Optional, Protocol, Self
from scrapling.core._types import Any, Protocol, Self
class XPathExpr(OriginalXPathExpr):
textnode: bool = False
attribute: Optional[str] = None
attribute: str | None = None
@classmethod
def from_xpath(
cls,
xpath: OriginalXPathExpr,
textnode: bool = False,
attribute: Optional[str] = None,
attribute: str | None = None,
) -> Self:
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
x.textnode = textnode
@@ -71,10 +70,10 @@ class XPathExpr(OriginalXPathExpr):
# e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator
class TranslatorProtocol(Protocol):
def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pragma: no cover
def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pyright: ignore # pragma: no cover
pass
def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pragma: no cover
def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pyright: ignore # pragma: no cover
pass
@@ -121,9 +120,15 @@ class TranslatorMixin:
class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
@lru_cache(maxsize=256)
def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
return super().css_to_xpath(css, prefix)
translator = HTMLTranslator()
# Using a function instead of the translator directly to avoid Pyright override error
@lru_cache(maxsize=256)
def css_to_xpath(query: str) -> str:
"""Return translated XPath version of a given CSS query"""
return translator.css_to_xpath(query)