diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index f157879..4670701 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -129,9 +129,8 @@ class TextHandlers(List[TextHandler]): class AttributesHandler(Mapping): - """A read-only mapping to use instead of the standard dictionary for the speed boost but - at the same time I use it to add more functionalities. - If standard dictionary is needed, just convert this class to dictionary with `dict` function + """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities. + If standard dictionary is needed, just convert this class to dictionary with `dict` function """ __slots__ = ('_data',) diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 41f5811..ec1b8ac 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -1,9 +1,11 @@ """ Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason... -To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format -which will be important in future releases but most importantly... - so you don't have to learn a new selectors/api method like what bs4 done with soupsieve :) -> if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement + +To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly... + +So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :) + + if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement """ import re diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index c106332..16e5350 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -23,7 +23,7 @@ class StaticEngine: @staticmethod def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict: """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and - finally generates a referer header that looks like if this request came from Google's search of the current URL's domain. + finally generates a referer header that looks like if this request came from Google's search of the current URL's domain. :param headers: Current headers in the request if the user passed any :param url: The Target URL. @@ -65,6 +65,7 @@ class StaticEngine: def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP GET request for you but with some added flavors. + :param url: Target url. :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search of this URL's domain. @@ -77,6 +78,7 @@ class StaticEngine: def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP POST request for you but with some added flavors. + :param url: Target url. :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search of this URL's domain. @@ -89,6 +91,7 @@ class StaticEngine: def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP DELETE request for you but with some added flavors. + :param url: Target url. :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search of this URL's domain. @@ -101,6 +104,7 @@ class StaticEngine: def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP PUT request for you but with some added flavors. + :param url: Target url. :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search of this URL's domain. diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index d056620..f0053d7 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -12,8 +12,8 @@ from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callab class ResponseEncoding: - DEFAULT_ENCODING = "utf-8" - ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} + __DEFAULT_ENCODING = "utf-8" + __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} @classmethod @cache(maxsize=None) @@ -43,17 +43,17 @@ class ResponseEncoding: """Determine the appropriate character encoding from a content-type header. The encoding is determined by these rules in order: - 1. If no content-type is provided, use UTF-8 - 2. If charset parameter is present, use that encoding - 3. If content-type is text/*, use ISO-8859-1 per HTTP/1.1 spec - 4. If content-type is application/json, use UTF-8 per RFC 4627 - 5. Default to UTF-8 if nothing else matches + 1. If no content-type is provided, use UTF-8 + 2. If charset parameter is present, use that encoding + 3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec + 4. If content-type is application/json, use UTF-8 per RFC 4627 + 5. Default to UTF-8 if nothing else matches :param content_type: Content-Type header value or None :return: String naming the character encoding """ if not content_type: - return cls.DEFAULT_ENCODING + return cls.__DEFAULT_ENCODING try: content_type, params = cls.__parse_content_type(content_type) @@ -65,16 +65,16 @@ class ResponseEncoding: return encoding # Apply content-type specific rules - if content_type in cls.ISO_8859_1_CONTENT_TYPES: + if content_type in cls.__ISO_8859_1_CONTENT_TYPES: return "ISO-8859-1" if content_type == "application/json": - return cls.DEFAULT_ENCODING + return cls.__DEFAULT_ENCODING - return cls.DEFAULT_ENCODING + return cls.__DEFAULT_ENCODING except (ValueError, LookupError, UnicodeEncodeError): - return cls.DEFAULT_ENCODING + return cls.__DEFAULT_ENCODING class Response(Adaptor): diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 294baca..6594a68 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -11,6 +11,7 @@ class Fetcher(BaseFetcher): """ def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP GET request for you but with some added flavors. + :param url: Target url. :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. @@ -24,6 +25,7 @@ class Fetcher(BaseFetcher): def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP POST request for you but with some added flavors. + :param url: Target url. :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. @@ -37,12 +39,14 @@ class Fetcher(BaseFetcher): def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP PUT request for you but with some added flavors. + :param url: Target url :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. + create a referer header as if this request came from Google's search of this URL's domain. :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. + :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs) @@ -50,6 +54,7 @@ class Fetcher(BaseFetcher): def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: """Make basic HTTP DELETE request for you but with some added flavors. + :param url: Target url :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. @@ -77,6 +82,7 @@ class StealthyFetcher(BaseFetcher): ) -> Response: """ Opens up a browser and do your request based on your chosen options below. + :param url: Target url. :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode. :param block_images: Prevent the loading of images through Firefox preferences. @@ -127,14 +133,15 @@ class PlayWrightFetcher(BaseFetcher): Using this Fetcher class, you can do requests with: - Vanilla Playwright without any modifications other than the ones you chose. - Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com - Some of the things stealth mode does include: - 1) Patches the CDP runtime fingerprint. - 2) Mimics some of the real browsers' properties by injecting several JS files and using custom options. - 3) Using custom flags on launch to hide Playwright even more and make it faster. - 4) Generates real browser's headers of the same type and same user OS then append it to the request. + Some of the things stealth mode does include: + 1) Patches the CDP runtime fingerprint. + 2) Mimics some of the real browsers' properties by injecting several JS files and using custom options. + 3) Using custom flags on launch to hide Playwright even more and make it faster. + 4) Generates real browser's headers of the same type and same user OS then append it to the request. - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it. - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option. - > Note that these are the main options with PlayWright but it can be mixed together. + + > Note that these are the main options with PlayWright but it can be mixed together. """ def fetch( self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, @@ -147,6 +154,7 @@ class PlayWrightFetcher(BaseFetcher): nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, ) -> Response: """Opens up a browser and do your request based on your chosen options below. + :param url: Target url. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.