refactor: huge change, many features/class got a better naming
- `Adaptor` became `Selector` - `Adaptors` became `Selectors` - `auto_match` argument/feature became `adaptive` - `adaptor_arguments` argument became `selector_config` - `automatch_domain` argument became `adaptive_domain` - `additional_arguments` argument became `additional_args` - `storage_adaptors` file became just `storage`
This commit is contained in:
Karim shoair
@@ -430,7 +430,7 @@ class ScraplingMCPServer:
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
additional_arguments: Optional[Dict] = None,
|
||||
additional_args: Optional[Dict] = None,
|
||||
) -> ResponseModel:
|
||||
"""Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.
|
||||
Note: This is best suitable for high protection levels. It's slower than the other tools.
|
||||
@@ -467,7 +467,7 @@ class ScraplingMCPServer:
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
page = await StealthyFetcher.async_fetch(
|
||||
url,
|
||||
@@ -491,7 +491,7 @@ class ScraplingMCPServer:
|
||||
solve_cloudflare=solve_cloudflare,
|
||||
disable_resources=disable_resources,
|
||||
wait_selector_state=wait_selector_state,
|
||||
additional_arguments=additional_arguments,
|
||||
additional_args=additional_args,
|
||||
)
|
||||
return _ContentTranslator(
|
||||
Convertor._extract_content(
|
||||
@@ -530,7 +530,7 @@ class ScraplingMCPServer:
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
additional_arguments: Optional[Dict] = None,
|
||||
additional_args: Optional[Dict] = None,
|
||||
) -> List[ResponseModel]:
|
||||
"""Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.
|
||||
Note: This is best suitable for high protection levels. It's slower than the other tools.
|
||||
@@ -567,7 +567,7 @@ class ScraplingMCPServer:
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
async with AsyncStealthySession(
|
||||
wait=wait,
|
||||
@@ -591,7 +591,7 @@ class ScraplingMCPServer:
|
||||
solve_cloudflare=solve_cloudflare,
|
||||
disable_resources=disable_resources,
|
||||
wait_selector_state=wait_selector_state,
|
||||
additional_arguments=additional_arguments,
|
||||
additional_args=additional_args,
|
||||
) as session:
|
||||
tasks = [session.fetch(url) for url in urls]
|
||||
responses = await gather(*tasks)
|
||||
|
||||
+17
-17
@@ -27,7 +27,7 @@ from orjson import loads as json_loads, JSONDecodeError
|
||||
from scrapling import __version__
|
||||
from scrapling.core.custom_types import TextHandler
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.parser import Adaptor, Adaptors
|
||||
from scrapling.parser import Selector, Selectors
|
||||
from scrapling.core._types import (
|
||||
List,
|
||||
Optional,
|
||||
@@ -399,9 +399,9 @@ class CurlParser:
|
||||
return None
|
||||
|
||||
|
||||
def show_page_in_browser(page: Adaptor):
|
||||
if not page or not isinstance(page, Adaptor):
|
||||
log.error("Input must be of type `Adaptor`")
|
||||
def show_page_in_browser(page: Selector):
|
||||
if not page or not isinstance(page, Selector):
|
||||
log.error("Input must be of type `Selector`")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -421,7 +421,7 @@ class CustomShell:
|
||||
def __init__(self, code, log_level="debug"):
|
||||
self.code = code
|
||||
self.page = None
|
||||
self.pages = Adaptors([])
|
||||
self.pages = Selectors([])
|
||||
self._curl_parser = CurlParser()
|
||||
log_level = log_level.strip().lower()
|
||||
|
||||
@@ -457,7 +457,7 @@ class CustomShell:
|
||||
- Fetcher/AsyncFetcher
|
||||
- DynamicFetcher
|
||||
- StealthyFetcher
|
||||
- Adaptor
|
||||
- Selector
|
||||
|
||||
-> Useful shortcuts:
|
||||
- {"get":<30} Shortcut for `Fetcher.get`
|
||||
@@ -469,7 +469,7 @@ class CustomShell:
|
||||
|
||||
-> Useful commands
|
||||
- {"page / response":<30} The response object of the last page you fetched
|
||||
- {"pages":<30} Adaptors object of the last 5 response objects you fetched
|
||||
- {"pages":<30} Selectors object of the last 5 response objects you fetched
|
||||
- {"uncurl('curl_command')":<30} Convert curl command to a Request object. (Optimized to handle curl commands copied from DevTools network tab.)
|
||||
- {"curl2fetcher('curl_command')":<30} Convert curl command and make the request with Fetcher. (Optimized to handle curl commands copied from DevTools network tab.)
|
||||
- {"view(page)":<30} View page in a browser
|
||||
@@ -481,7 +481,7 @@ Type 'exit' or press Ctrl+D to exit.
|
||||
def update_page(self, result):
|
||||
"""Update the current page and add to pages history"""
|
||||
self.page = result
|
||||
if isinstance(result, (Response, Adaptor)):
|
||||
if isinstance(result, (Response, Selector)):
|
||||
self.pages.append(result)
|
||||
if len(self.pages) > 5:
|
||||
self.pages.pop(0) # Remove oldest item
|
||||
@@ -528,7 +528,7 @@ Type 'exit' or press Ctrl+D to exit.
|
||||
"DynamicFetcher": DynamicFetcher,
|
||||
"stealthy_fetch": stealthy_fetch,
|
||||
"StealthyFetcher": StealthyFetcher,
|
||||
"Adaptor": Adaptor,
|
||||
"Selector": Selector,
|
||||
"page": self.page,
|
||||
"response": self.page,
|
||||
"pages": self.pages,
|
||||
@@ -586,14 +586,14 @@ class Convertor:
|
||||
@classmethod
|
||||
def _extract_content(
|
||||
cls,
|
||||
page: Adaptor,
|
||||
page: Selector,
|
||||
extraction_type: extraction_types = "markdown",
|
||||
css_selector: Optional[str] = None,
|
||||
main_content_only: bool = False,
|
||||
) -> Generator[str, None, None]:
|
||||
"""Extract the content of an Adaptor"""
|
||||
if not page or not isinstance(page, Adaptor):
|
||||
raise TypeError("Input must be of type `Adaptor`")
|
||||
"""Extract the content of an Selector"""
|
||||
if not page or not isinstance(page, Selector):
|
||||
raise TypeError("Input must be of type `Selector`")
|
||||
elif not extraction_type or extraction_type not in cls._extension_map.values():
|
||||
raise ValueError(f"Unknown extraction type: {extraction_type}")
|
||||
else:
|
||||
@@ -622,11 +622,11 @@ class Convertor:
|
||||
|
||||
@classmethod
|
||||
def write_content_to_file(
|
||||
cls, page: Adaptor, filename: str, css_selector: Optional[str] = None
|
||||
cls, page: Selector, filename: str, css_selector: Optional[str] = None
|
||||
) -> None:
|
||||
"""Write an Adaptor's content to a file"""
|
||||
if not page or not isinstance(page, Adaptor):
|
||||
raise TypeError("Input must be of type `Adaptor`")
|
||||
"""Write an Selector's content to a file"""
|
||||
if not page or not isinstance(page, Selector):
|
||||
raise TypeError("Input must be of type `Selector`")
|
||||
elif not filename or not isinstance(filename, str) or not filename.strip():
|
||||
raise ValueError("Filename must be provided")
|
||||
elif not filename.endswith((".md", ".html", ".txt")):
|
||||
|
||||
Reference in New Issue
Block a user