fix: make MCP get schemas validator-safe
Use JSON-object input types for cookies and basic-auth fields in get and bulk_get so strict MCP schema validators can register tools reliably. Normalize auth dictionaries to the tuple format expected by fetchers to preserve runtime behavior.
This commit is contained in:
+34
-14
@@ -42,6 +42,20 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
|
|||||||
return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
|
return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
|
||||||
|
|
||||||
|
|
||||||
|
def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]:
|
||||||
|
"""Convert a credentials dictionary to a tuple accepted by fetchers."""
|
||||||
|
if not credentials:
|
||||||
|
return None
|
||||||
|
|
||||||
|
username = credentials.get("username")
|
||||||
|
password = credentials.get("password")
|
||||||
|
|
||||||
|
if username is None or password is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
class ScraplingMCPServer:
|
class ScraplingMCPServer:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get(
|
def get(
|
||||||
@@ -52,15 +66,15 @@ class ScraplingMCPServer:
|
|||||||
main_content_only: bool = True,
|
main_content_only: bool = True,
|
||||||
params: Optional[Dict | List | Tuple] = None,
|
params: Optional[Dict | List | Tuple] = None,
|
||||||
headers: Optional[Mapping[str, Optional[str]]] = None,
|
headers: Optional[Mapping[str, Optional[str]]] = None,
|
||||||
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
|
cookies: Optional[Dict[str, str]] = None,
|
||||||
timeout: Optional[int | float] = 30,
|
timeout: Optional[int | float] = 30,
|
||||||
follow_redirects: bool = True,
|
follow_redirects: bool = True,
|
||||||
max_redirects: int = 30,
|
max_redirects: int = 30,
|
||||||
retries: Optional[int] = 3,
|
retries: Optional[int] = 3,
|
||||||
retry_delay: Optional[int] = 1,
|
retry_delay: Optional[int] = 1,
|
||||||
proxy: Optional[str] = None,
|
proxy: Optional[str] = None,
|
||||||
proxy_auth: Optional[Tuple[str, str]] = None,
|
proxy_auth: Optional[Dict[str, str]] = None,
|
||||||
auth: Optional[Tuple[str, str]] = None,
|
auth: Optional[Dict[str, str]] = None,
|
||||||
verify: Optional[bool] = True,
|
verify: Optional[bool] = True,
|
||||||
http3: Optional[bool] = False,
|
http3: Optional[bool] = False,
|
||||||
stealthy_headers: Optional[bool] = True,
|
stealthy_headers: Optional[bool] = True,
|
||||||
@@ -87,20 +101,23 @@ class ScraplingMCPServer:
|
|||||||
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||||
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
|
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
|
||||||
Cannot be used together with the `proxies` parameter.
|
Cannot be used together with the `proxies` parameter.
|
||||||
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
|
:param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
|
||||||
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
|
:param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
:param verify: Whether to verify HTTPS certificates.
|
||||||
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
|
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
|
||||||
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
|
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
|
||||||
"""
|
"""
|
||||||
|
normalized_proxy_auth = _NormalizeCredentials(proxy_auth)
|
||||||
|
normalized_auth = _NormalizeCredentials(auth)
|
||||||
|
|
||||||
page = Fetcher.get(
|
page = Fetcher.get(
|
||||||
url,
|
url,
|
||||||
auth=auth,
|
auth=normalized_auth,
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
http3=http3,
|
http3=http3,
|
||||||
verify=verify,
|
verify=verify,
|
||||||
params=params,
|
params=params,
|
||||||
proxy_auth=proxy_auth,
|
proxy_auth=normalized_proxy_auth,
|
||||||
retry_delay=retry_delay,
|
retry_delay=retry_delay,
|
||||||
stealthy_headers=stealthy_headers,
|
stealthy_headers=stealthy_headers,
|
||||||
impersonate=impersonate,
|
impersonate=impersonate,
|
||||||
@@ -130,15 +147,15 @@ class ScraplingMCPServer:
|
|||||||
main_content_only: bool = True,
|
main_content_only: bool = True,
|
||||||
params: Optional[Dict | List | Tuple] = None,
|
params: Optional[Dict | List | Tuple] = None,
|
||||||
headers: Optional[Mapping[str, Optional[str]]] = None,
|
headers: Optional[Mapping[str, Optional[str]]] = None,
|
||||||
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
|
cookies: Optional[Dict[str, str]] = None,
|
||||||
timeout: Optional[int | float] = 30,
|
timeout: Optional[int | float] = 30,
|
||||||
follow_redirects: bool = True,
|
follow_redirects: bool = True,
|
||||||
max_redirects: int = 30,
|
max_redirects: int = 30,
|
||||||
retries: Optional[int] = 3,
|
retries: Optional[int] = 3,
|
||||||
retry_delay: Optional[int] = 1,
|
retry_delay: Optional[int] = 1,
|
||||||
proxy: Optional[str] = None,
|
proxy: Optional[str] = None,
|
||||||
proxy_auth: Optional[Tuple[str, str]] = None,
|
proxy_auth: Optional[Dict[str, str]] = None,
|
||||||
auth: Optional[Tuple[str, str]] = None,
|
auth: Optional[Dict[str, str]] = None,
|
||||||
verify: Optional[bool] = True,
|
verify: Optional[bool] = True,
|
||||||
http3: Optional[bool] = False,
|
http3: Optional[bool] = False,
|
||||||
stealthy_headers: Optional[bool] = True,
|
stealthy_headers: Optional[bool] = True,
|
||||||
@@ -165,17 +182,20 @@ class ScraplingMCPServer:
|
|||||||
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||||
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
|
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
|
||||||
Cannot be used together with the `proxies` parameter.
|
Cannot be used together with the `proxies` parameter.
|
||||||
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
|
:param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
|
||||||
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
|
:param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
:param verify: Whether to verify HTTPS certificates.
|
||||||
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
|
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
|
||||||
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
|
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
|
||||||
"""
|
"""
|
||||||
|
normalized_proxy_auth = _NormalizeCredentials(proxy_auth)
|
||||||
|
normalized_auth = _NormalizeCredentials(auth)
|
||||||
|
|
||||||
async with FetcherSession() as session:
|
async with FetcherSession() as session:
|
||||||
tasks: List[Any] = [
|
tasks: List[Any] = [
|
||||||
session.get(
|
session.get(
|
||||||
url,
|
url,
|
||||||
auth=auth,
|
auth=normalized_auth,
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
http3=http3,
|
http3=http3,
|
||||||
verify=verify,
|
verify=verify,
|
||||||
@@ -184,7 +204,7 @@ class ScraplingMCPServer:
|
|||||||
cookies=cookies,
|
cookies=cookies,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
retries=retries,
|
retries=retries,
|
||||||
proxy_auth=proxy_auth,
|
proxy_auth=normalized_proxy_auth,
|
||||||
retry_delay=retry_delay,
|
retry_delay=retry_delay,
|
||||||
impersonate=impersonate,
|
impersonate=impersonate,
|
||||||
max_redirects=max_redirects,
|
max_redirects=max_redirects,
|
||||||
|
|||||||
Reference in New Issue
Block a user