diff --git a/README.md b/README.md index f344704..9004a2a 100644 --- a/README.md +++ b/README.md @@ -24,14 +24,6 @@

- - Installation - - · - - Overview - - · Selection methods @@ -40,6 +32,14 @@ Choosing a fetcher · + + CLI + + · + + MCP mode + + · Migrating from Beautifulsoup @@ -69,6 +69,7 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet + diff --git a/docs/index.md b/docs/index.md index 51d490d..f2c5198 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,6 +38,7 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet

+
diff --git a/images/decodo.png b/images/decodo.png new file mode 100644 index 0000000..c06ca11 Binary files /dev/null and b/images/decodo.png differ diff --git a/scrapling/__init__.py b/scrapling/__init__.py index d7de2f8..99e72b7 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.3" +__version__ = "0.3.4" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 8a38391..58ec258 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -317,7 +317,7 @@ def show_page_in_browser(page: Selector): # pragma: no cover try: fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html") - with open(fd, "w", encoding=page.encoding) as f: + with open(fd, "wb") as f: f.write(page.body) open_in_browser(f"file://{fname}") @@ -335,15 +335,25 @@ class CustomShell: from scrapling.fetchers import ( Fetcher as __Fetcher, AsyncFetcher as __AsyncFetcher, + FetcherSession as __FetcherSession, DynamicFetcher as __DynamicFetcher, + DynamicSession as __DynamicSession, + AsyncDynamicSession as __AsyncDynamicSession, StealthyFetcher as __StealthyFetcher, + StealthySession as __StealthySession, + AsyncStealthySession as __AsyncStealthySession, ) self.__InteractiveShellEmbed = __InteractiveShellEmbed self.__Fetcher = __Fetcher self.__AsyncFetcher = __AsyncFetcher + self.__FetcherSession = __FetcherSession self.__DynamicFetcher = __DynamicFetcher + self.__DynamicSession = __DynamicSession + self.__AsyncDynamicSession = __AsyncDynamicSession self.__StealthyFetcher = __StealthyFetcher + self.__StealthySession = __StealthySession + self.__AsyncStealthySession = __AsyncStealthySession self.code = code self.page = None self.pages = Selectors([]) @@ -379,9 +389,9 @@ class CustomShell: """Create a custom banner for the shell""" return f""" -> Available Scrapling objects: - - Fetcher/AsyncFetcher - - DynamicFetcher - - StealthyFetcher + - Fetcher/AsyncFetcher/FetcherSession + - DynamicFetcher/DynamicSession/AsyncDynamicSession + - StealthyFetcher/StealthySession/AsyncStealthySession - Selector -> Useful shortcuts: @@ -449,6 +459,11 @@ Type 'exit' or press Ctrl+D to exit. "delete": delete, "Fetcher": self.__Fetcher, "AsyncFetcher": self.__AsyncFetcher, + "FetcherSession": self.__FetcherSession, + "DynamicSession": self.__DynamicSession, + "AsyncDynamicSession": self.__AsyncDynamicSession, + "StealthySession": self.__StealthySession, + "AsyncStealthySession": self.__AsyncStealthySession, "fetch": dynamic_fetch, "DynamicFetcher": self.__DynamicFetcher, "stealthy_fetch": stealthy_fetch, diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 00c5567..8e94559 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -31,7 +31,7 @@ class SyncSession: def __init__(self, max_pages: int = 1): self.max_pages = max_pages self.page_pool = PagePool(max_pages) - self.__max_wait_for_page = 60 + self._max_wait_for_page = 60 self.playwright: Optional[Playwright] = None self.context: Optional[BrowserContext] = None self._closed = False @@ -50,7 +50,7 @@ class SyncSession: # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() - while time() - start_time < self.__max_wait_for_page: + while time() - start_time < self._max_wait_for_page: # Wait for any pages to finish, then clean them up sleep(0.05) self.page_pool.close_all_finished_pages() @@ -58,7 +58,7 @@ class SyncSession: break else: raise TimeoutError( - f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" ) page = self.context.new_page() @@ -111,7 +111,7 @@ class AsyncSession(SyncSession): # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() - while time() - start_time < self.__max_wait_for_page: + while time() - start_time < self._max_wait_for_page: # Wait for any pages to finish, then clean them up await asyncio_sleep(0.05) await self.page_pool.aclose_all_finished_pages() @@ -119,7 +119,7 @@ class AsyncSession(SyncSession): break else: raise TimeoutError( - f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" ) page = await self.context.new_page() diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index ad515ca..a4ca6f0 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -14,6 +14,7 @@ from playwright.async_api import ( Locator as AsyncLocator, Page as async_Page, ) +from playwright._impl._errors import Error as PlaywrightError from ._validators import validate, CamoufoxConfig from ._base import SyncSession, AsyncSession, StealthySessionMixin @@ -201,20 +202,34 @@ class StealthySession(StealthySessionMixin, SyncSession): self._closed = True + @staticmethod + def _get_page_content(page: Page) -> str | None: + """ + A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + :param page: The page to extract content from. + :return: + """ + while True: + try: + return page.content() or "" + except PlaywrightError: + page.wait_for_timeout(1000) + continue + def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed :param page: The targeted page :return: """ - challenge_type = self._detect_cloudflare(page.content()) + challenge_type = self._detect_cloudflare(self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") return else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": - while "Just a moment..." in (page.content()): + while "Just a moment..." in (self._get_page_content(page)): log.info("Waiting for Cloudflare wait page to disappear.") page.wait_for_timeout(1000) page.wait_for_load_state() @@ -222,7 +237,7 @@ class StealthySession(StealthySessionMixin, SyncSession): return else: - while "Verifying you are human." in page.content(): + while "Verifying you are human." in self._get_page_content(page): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared page.wait_for_timeout(500) @@ -506,20 +521,34 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): self._closed = True + @staticmethod + async def _get_page_content(page: async_Page) -> str | None: + """ + A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + :param page: The page to extract content from. + :return: + """ + while True: + try: + return (await page.content()) or "" + except PlaywrightError: + await page.wait_for_timeout(1000) + continue + async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version :param page: The async targeted page :return: """ - challenge_type = self._detect_cloudflare(await page.content()) + challenge_type = self._detect_cloudflare(await self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") return else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": # pragma: no cover - while "Just a moment..." in (await page.content()): + while "Just a moment..." in (await self._get_page_content(page)): log.info("Waiting for Cloudflare wait page to disappear.") await page.wait_for_timeout(1000) await page.wait_for_load_state() @@ -527,7 +556,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): return else: - while "Verifying you are human." in (await page.content()): + while "Verifying you are human." in (await self._get_page_content(page)): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared await page.wait_for_timeout(500) diff --git a/scrapling/parser.py b/scrapling/parser.py index 409e09d..4fb510c 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -339,7 +339,10 @@ class Selector(SelectorsGeneration): @property def html_content(self) -> TextHandler: """Return the inner HTML code of the element""" - return TextHandler(tostring(self._root, encoding=self.encoding, method="html", with_tail=False)) + content = tostring(self._root, encoding=self.encoding, method="html", with_tail=False) + if isinstance(content, bytes): + content = content.decode("utf-8") + return TextHandler(content) @property def body(self): @@ -348,15 +351,16 @@ class Selector(SelectorsGeneration): def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" - return TextHandler( - tostring( - self._root, - encoding=self.encoding, - pretty_print=True, - method="html", - with_tail=False, - ) + content = tostring( + self._root, + encoding=self.encoding, + pretty_print=True, + method="html", + with_tail=False, ) + if isinstance(content, bytes): + content = content.decode("utf-8") + return TextHandler(content) def has_class(self, class_name: str) -> bool: """Check if the element has a specific class diff --git a/setup.cfg b/setup.cfg index d6ecc6c..5e52c3f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.3 +version = 0.3.4 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!