fix: improve type hints for the static checkers
This commit is contained in:
@@ -28,6 +28,7 @@ from scrapling.engines.toolbelt.navigation import (
|
|||||||
)
|
)
|
||||||
from scrapling.core._types import (
|
from scrapling.core._types import (
|
||||||
Any,
|
Any,
|
||||||
|
Awaitable,
|
||||||
Dict,
|
Dict,
|
||||||
List,
|
List,
|
||||||
Set,
|
Set,
|
||||||
@@ -152,7 +153,7 @@ class SyncSession:
|
|||||||
response_container: List,
|
response_container: List,
|
||||||
xhr_pattern: Optional[str] = None,
|
xhr_pattern: Optional[str] = None,
|
||||||
xhr_container: Optional[List] = None,
|
xhr_container: Optional[List] = None,
|
||||||
) -> Callable:
|
) -> Callable[[SyncPlaywrightResponse], None]:
|
||||||
"""Create a response handler that captures the final navigation response and optionally XHR/fetch responses.
|
"""Create a response handler that captures the final navigation response and optionally XHR/fetch responses.
|
||||||
|
|
||||||
:param page_info: The PageInfo object containing the page
|
:param page_info: The PageInfo object containing the page
|
||||||
@@ -162,7 +163,7 @@ class SyncSession:
|
|||||||
:return: A callback function for page.on("response", ...)
|
:return: A callback function for page.on("response", ...)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def handle_response(finished_response: SyncPlaywrightResponse):
|
def handle_response(finished_response: SyncPlaywrightResponse) -> None:
|
||||||
if (
|
if (
|
||||||
finished_response.request.resource_type == "document"
|
finished_response.request.resource_type == "document"
|
||||||
and finished_response.request.is_navigation_request()
|
and finished_response.request.is_navigation_request()
|
||||||
@@ -337,7 +338,7 @@ class AsyncSession:
|
|||||||
response_container: List,
|
response_container: List,
|
||||||
xhr_pattern: Optional[str] = None,
|
xhr_pattern: Optional[str] = None,
|
||||||
xhr_container: Optional[List] = None,
|
xhr_container: Optional[List] = None,
|
||||||
) -> Callable:
|
) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]:
|
||||||
"""Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
|
"""Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
|
||||||
|
|
||||||
:param page_info: The PageInfo object containing the page
|
:param page_info: The PageInfo object containing the page
|
||||||
@@ -347,7 +348,7 @@ class AsyncSession:
|
|||||||
:return: A callback function for page.on("response", ...)
|
:return: A callback function for page.on("response", ...)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def handle_response(finished_response: AsyncPlaywrightResponse):
|
async def handle_response(finished_response: AsyncPlaywrightResponse) -> None:
|
||||||
if (
|
if (
|
||||||
finished_response.request.resource_type == "document"
|
finished_response.request.resource_type == "document"
|
||||||
and finished_response.request.is_navigation_request()
|
and finished_response.request.is_navigation_request()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from playwright.async_api import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from scrapling.core.utils import log
|
from scrapling.core.utils import log
|
||||||
from scrapling.core._types import Optional, ProxyType, Unpack
|
from scrapling.core._types import Optional, List, ProxyType, Unpack
|
||||||
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
|
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
|
||||||
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
||||||
from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams
|
from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams
|
||||||
@@ -139,7 +139,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
|
|||||||
with self._page_generator(
|
with self._page_generator(
|
||||||
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
||||||
) as page_info:
|
) as page_info:
|
||||||
final_response, xhr_captured = [None], []
|
final_response: List = [None]
|
||||||
|
xhr_captured: List = []
|
||||||
page = page_info.page
|
page = page_info.page
|
||||||
page.on(
|
page.on(
|
||||||
"response",
|
"response",
|
||||||
@@ -319,7 +320,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
|
|||||||
async with self._page_generator(
|
async with self._page_generator(
|
||||||
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
||||||
) as page_info:
|
) as page_info:
|
||||||
final_response, xhr_captured = [None], []
|
final_response: List = [None]
|
||||||
|
xhr_captured: List = []
|
||||||
page = page_info.page
|
page = page_info.page
|
||||||
page.on(
|
page.on(
|
||||||
"response",
|
"response",
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from patchright.sync_api import sync_playwright
|
|||||||
from patchright.async_api import async_playwright
|
from patchright.async_api import async_playwright
|
||||||
|
|
||||||
from scrapling.core.utils import log
|
from scrapling.core.utils import log
|
||||||
from scrapling.core._types import Any, Optional, ProxyType, Unpack
|
from scrapling.core._types import Any, List, Optional, ProxyType, Unpack
|
||||||
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
|
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
|
||||||
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
|
||||||
from scrapling.engines._browsers._types import StealthSession, StealthFetchParams
|
from scrapling.engines._browsers._types import StealthSession, StealthFetchParams
|
||||||
@@ -222,7 +222,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
|
|||||||
with self._page_generator(
|
with self._page_generator(
|
||||||
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
||||||
) as page_info:
|
) as page_info:
|
||||||
final_response, xhr_captured = [None], []
|
final_response: List = [None]
|
||||||
|
xhr_captured: List = []
|
||||||
page = page_info.page
|
page = page_info.page
|
||||||
page.on(
|
page.on(
|
||||||
"response",
|
"response",
|
||||||
@@ -489,7 +490,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
|||||||
async with self._page_generator(
|
async with self._page_generator(
|
||||||
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
||||||
) as page_info:
|
) as page_info:
|
||||||
final_response, xhr_captured = [None], []
|
final_response: List = [None]
|
||||||
|
xhr_captured: List = []
|
||||||
page = page_info.page
|
page = page_info.page
|
||||||
page.on(
|
page.on(
|
||||||
"response",
|
"response",
|
||||||
|
|||||||
Reference in New Issue
Block a user