fix(mcp): remove unneeded code and fix type hint for mypy

This commit is contained in:
Karim shoair
2026-03-28 17:19:42 +02:00
parent c458ab65a2
commit 0f6dcccf5f
+12 -26
View File
@@ -17,6 +17,7 @@ from scrapling.fetchers import (
from scrapling.core._types import ( from scrapling.core._types import (
Optional, Optional,
Literal, Literal,
Union,
Tuple, Tuple,
Mapping, Mapping,
Dict, Dict,
@@ -100,28 +101,6 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu
return username, password return username, password
def _build_fetch_kwargs(**params) -> Dict[str, Any]:
"""Build kwargs dict for session.fetch() from request-level parameters."""
kwargs: Dict[str, Any] = {}
# These are the params that session.fetch() accepts as per-request overrides
request_level_keys = (
"wait",
"timeout",
"google_search",
"extra_headers",
"disable_resources",
"wait_selector",
"wait_selector_state",
"network_idle",
"proxy",
"solve_cloudflare",
)
for key in request_level_keys:
if key in params and params[key] is not None:
kwargs[key] = params[key]
return kwargs
class ScraplingMCPServer: class ScraplingMCPServer:
def __init__(self): def __init__(self):
self._sessions: Dict[str, _SessionEntry] = {} self._sessions: Dict[str, _SessionEntry] = {}
@@ -215,6 +194,7 @@ class ScraplingMCPServer:
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
) )
session: Union[AsyncDynamicSession, AsyncStealthySession]
if session_type == "stealthy": if session_type == "stealthy":
session = AsyncStealthySession( session = AsyncStealthySession(
**common_kwargs, **common_kwargs,
@@ -563,7 +543,9 @@ class ScraplingMCPServer:
""" """
if session_id: if session_id:
entry = self._get_session(session_id, "dynamic") entry = self._get_session(session_id, "dynamic")
fetch_kwargs = _build_fetch_kwargs( tasks = [
entry.session.fetch(
url,
wait=wait, wait=wait,
timeout=timeout, timeout=timeout,
google_search=google_search, google_search=google_search,
@@ -574,7 +556,8 @@ class ScraplingMCPServer:
network_idle=network_idle, network_idle=network_idle,
proxy=proxy, proxy=proxy,
) )
tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] for url in urls
]
responses = await gather(*tasks) responses = await gather(*tasks)
else: else:
async with AsyncDynamicSession( async with AsyncDynamicSession(
@@ -767,7 +750,9 @@ class ScraplingMCPServer:
""" """
if session_id: if session_id:
entry = self._get_session(session_id, "stealthy") entry = self._get_session(session_id, "stealthy")
fetch_kwargs = _build_fetch_kwargs( tasks = [
entry.session.fetch(
url,
wait=wait, wait=wait,
timeout=timeout, timeout=timeout,
google_search=google_search, google_search=google_search,
@@ -779,7 +764,8 @@ class ScraplingMCPServer:
proxy=proxy, proxy=proxy,
solve_cloudflare=solve_cloudflare, solve_cloudflare=solve_cloudflare,
) )
tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] for url in urls
]
responses = await gather(*tasks) responses = await gather(*tasks)
else: else:
async with AsyncStealthySession( async with AsyncStealthySession(