diff --git a/README.md b/README.md index 3838b1f..8d1c778 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Tired of your PC slowing you down? Can’t keep your machine on 24/7 for scrapin ```python from scrapling.fetchers import Fetcher -# Do HTTP GET request to a web page and create an Selector instance +# Do HTTP GET request to a web page and create a Selector instance page = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True) # Get all text content from all HTML tags in the page except the `script` and `style` tags page.get_all_text(ignore_tags=('script', 'style')) diff --git a/scrapling/cli.py b/scrapling/cli.py index e5e91d5..940c27d 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -3,6 +3,7 @@ from subprocess import check_output from sys import executable as python_executable from scrapling.core.utils import log +from scrapling.engines.toolbelt import Response from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders @@ -32,12 +33,12 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any def __Request_and_Save( - fetcher_func: Callable, + fetcher_func: Callable[..., Response], url: str, output_file: str, css_selector: Optional[str] = None, **kwargs, -): +) -> None: """Make a request using the specified fetcher function and save the result""" # Handle relative paths - convert to an absolute path based on the current working directory output_path = Path(output_file) diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index a41e077..85e7013 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -4,6 +4,7 @@ Type definitions for type checking purposes. from typing import ( TYPE_CHECKING, + cast, overload, Any, Callable, @@ -32,8 +33,13 @@ extraction_types = Literal["text", "html", "markdown"] StrOrBytes = Union[str, bytes] -if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self -else: - Self = object +try: + # Python 3.11+ + from typing import Self # novermin +except ImportError: + try: + from typing_extensions import Self # Backport + except ImportError: + from typing import TypeVar + + Self = object diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 3d523a3..ccbe536 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -63,7 +63,7 @@ class ScraplingMCPServer: main_content_only: bool = True, params: Optional[Union[Dict, List, Tuple]] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None, + cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None, timeout: Optional[Union[int, float]] = 30, follow_redirects: bool = True, max_redirects: int = 30, @@ -142,7 +142,7 @@ class ScraplingMCPServer: main_content_only: bool = True, params: Optional[Union[Dict, List, Tuple]] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None, + cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None, timeout: Optional[Union[int, float]] = 30, follow_redirects: bool = True, max_redirects: int = 30, diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py index 22859b9..afad094 100644 --- a/scrapling/core/mixins.py +++ b/scrapling/core/mixins.py @@ -1,9 +1,13 @@ class SelectorsGeneration: - """Selectors generation functions + """ + Functions for generating selectors Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm - Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591""" + Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591 + """ - def __general_selection(self, selection: str = "css", full_path=False) -> str: + def __general_selection( + self, selection: str = "css", full_path: bool = False + ) -> str: """Generate a selector for the current element. :return: A string of the generated selector. """ @@ -80,7 +84,7 @@ class SelectorsGeneration: @property def generate_xpath_selector(self) -> str: - """Generate a XPath selector for the current element + """Generate an XPath selector for the current element :return: A string of the generated selector. """ return self.__general_selection("xpath") diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index b100ad3..f11f386 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -570,7 +570,7 @@ Type 'exit' or press Ctrl+D to exit. class Convertor: """Utils for the extract shell command""" - _extension_map: dict[str, extraction_types] = { + _extension_map: Dict[str, extraction_types] = { "md": "markdown", "html": "html", "txt": "text", @@ -591,7 +591,7 @@ class Convertor: css_selector: Optional[str] = None, main_content_only: bool = False, ) -> Generator[str, None, None]: - """Extract the content of an Selector""" + """Extract the content of a Selector""" if not page or not isinstance(page, Selector): raise TypeError("Input must be of type `Selector`") elif not extraction_type or extraction_type not in cls._extension_map.values(): @@ -624,7 +624,7 @@ class Convertor: def write_content_to_file( cls, page: Selector, filename: str, css_selector: Optional[str] = None ) -> None: - """Write an Selector's content to a file""" + """Write a Selector's content to a file""" if not page or not isinstance(page, Selector): raise TypeError("Input must be of type `Selector`") elif not filename or not isinstance(filename, str) or not filename.strip(): diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 9250ab6..9c987ff 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -1,11 +1,11 @@ """ -Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason... +Most of this file is an adapted version of the parsel library's translator with some modifications simply for 1 important reason... -To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly... +To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match the Parsel/Scrapy selectors format which will be important in future releases but most importantly... So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :) - if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement + If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement """ import re