fix(mcp): remove unneeded code and fix type hint for mypy
This commit is contained in:
+33
-47
@@ -17,6 +17,7 @@ from scrapling.fetchers import (
|
|||||||
from scrapling.core._types import (
|
from scrapling.core._types import (
|
||||||
Optional,
|
Optional,
|
||||||
Literal,
|
Literal,
|
||||||
|
Union,
|
||||||
Tuple,
|
Tuple,
|
||||||
Mapping,
|
Mapping,
|
||||||
Dict,
|
Dict,
|
||||||
@@ -100,28 +101,6 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu
|
|||||||
return username, password
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
def _build_fetch_kwargs(**params) -> Dict[str, Any]:
|
|
||||||
"""Build kwargs dict for session.fetch() from request-level parameters."""
|
|
||||||
kwargs: Dict[str, Any] = {}
|
|
||||||
# These are the params that session.fetch() accepts as per-request overrides
|
|
||||||
request_level_keys = (
|
|
||||||
"wait",
|
|
||||||
"timeout",
|
|
||||||
"google_search",
|
|
||||||
"extra_headers",
|
|
||||||
"disable_resources",
|
|
||||||
"wait_selector",
|
|
||||||
"wait_selector_state",
|
|
||||||
"network_idle",
|
|
||||||
"proxy",
|
|
||||||
"solve_cloudflare",
|
|
||||||
)
|
|
||||||
for key in request_level_keys:
|
|
||||||
if key in params and params[key] is not None:
|
|
||||||
kwargs[key] = params[key]
|
|
||||||
return kwargs
|
|
||||||
|
|
||||||
|
|
||||||
class ScraplingMCPServer:
|
class ScraplingMCPServer:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._sessions: Dict[str, _SessionEntry] = {}
|
self._sessions: Dict[str, _SessionEntry] = {}
|
||||||
@@ -215,6 +194,7 @@ class ScraplingMCPServer:
|
|||||||
wait_selector_state=wait_selector_state,
|
wait_selector_state=wait_selector_state,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
session: Union[AsyncDynamicSession, AsyncStealthySession]
|
||||||
if session_type == "stealthy":
|
if session_type == "stealthy":
|
||||||
session = AsyncStealthySession(
|
session = AsyncStealthySession(
|
||||||
**common_kwargs,
|
**common_kwargs,
|
||||||
@@ -563,18 +543,21 @@ class ScraplingMCPServer:
|
|||||||
"""
|
"""
|
||||||
if session_id:
|
if session_id:
|
||||||
entry = self._get_session(session_id, "dynamic")
|
entry = self._get_session(session_id, "dynamic")
|
||||||
fetch_kwargs = _build_fetch_kwargs(
|
tasks = [
|
||||||
wait=wait,
|
entry.session.fetch(
|
||||||
timeout=timeout,
|
url,
|
||||||
google_search=google_search,
|
wait=wait,
|
||||||
extra_headers=extra_headers,
|
timeout=timeout,
|
||||||
disable_resources=disable_resources,
|
google_search=google_search,
|
||||||
wait_selector=wait_selector,
|
extra_headers=extra_headers,
|
||||||
wait_selector_state=wait_selector_state,
|
disable_resources=disable_resources,
|
||||||
network_idle=network_idle,
|
wait_selector=wait_selector,
|
||||||
proxy=proxy,
|
wait_selector_state=wait_selector_state,
|
||||||
)
|
network_idle=network_idle,
|
||||||
tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls]
|
proxy=proxy,
|
||||||
|
)
|
||||||
|
for url in urls
|
||||||
|
]
|
||||||
responses = await gather(*tasks)
|
responses = await gather(*tasks)
|
||||||
else:
|
else:
|
||||||
async with AsyncDynamicSession(
|
async with AsyncDynamicSession(
|
||||||
@@ -767,19 +750,22 @@ class ScraplingMCPServer:
|
|||||||
"""
|
"""
|
||||||
if session_id:
|
if session_id:
|
||||||
entry = self._get_session(session_id, "stealthy")
|
entry = self._get_session(session_id, "stealthy")
|
||||||
fetch_kwargs = _build_fetch_kwargs(
|
tasks = [
|
||||||
wait=wait,
|
entry.session.fetch(
|
||||||
timeout=timeout,
|
url,
|
||||||
google_search=google_search,
|
wait=wait,
|
||||||
extra_headers=extra_headers,
|
timeout=timeout,
|
||||||
disable_resources=disable_resources,
|
google_search=google_search,
|
||||||
wait_selector=wait_selector,
|
extra_headers=extra_headers,
|
||||||
wait_selector_state=wait_selector_state,
|
disable_resources=disable_resources,
|
||||||
network_idle=network_idle,
|
wait_selector=wait_selector,
|
||||||
proxy=proxy,
|
wait_selector_state=wait_selector_state,
|
||||||
solve_cloudflare=solve_cloudflare,
|
network_idle=network_idle,
|
||||||
)
|
proxy=proxy,
|
||||||
tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls]
|
solve_cloudflare=solve_cloudflare,
|
||||||
|
)
|
||||||
|
for url in urls
|
||||||
|
]
|
||||||
responses = await gather(*tasks)
|
responses = await gather(*tasks)
|
||||||
else:
|
else:
|
||||||
async with AsyncStealthySession(
|
async with AsyncStealthySession(
|
||||||
|
|||||||
Reference in New Issue
Block a user