freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style(Fetchers): Setting correct type hint for wait_selector_state argument

Browse Source
This commit is contained in:
Karim shoair
2024-12-19 13:22:12 +02:00
parent 94745b1555
commit 5be8c2a95e
4 changed files with 14 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_types.py
+2
View File
@@ -5,6 +5,8 @@ Type definitions for type checking purposes.
from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
List, Literal, Optional, Pattern, Tuple, Type, Union) List, Literal, Optional, Pattern, Tuple, Type, Union)
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
try: try:
from typing import Protocol from typing import Protocol
except ImportError: except ImportError:
scrapling/engines/camo.py
+2 -2
View File
@@ -3,7 +3,7 @@ from camoufox.async_api import AsyncCamoufox
from camoufox.sync_api import Camoufox from camoufox.sync_api import Camoufox
from scrapling.core._types import (Callable, Dict, List, Literal, Optional, from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
Union) SelectorWaitStates, Union)
from scrapling.core.utils import log from scrapling.core.utils import log
from scrapling.engines.toolbelt import (Response, StatusText, from scrapling.engines.toolbelt import (Response, StatusText,
async_intercept_route, async_intercept_route,
@@ -18,7 +18,7 @@ class CamoufoxEngine:
self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False, self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True, block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: Optional[SelectorWaitStates] = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True,
geoip: Optional[bool] = False, geoip: Optional[bool] = False,
adaptor_arguments: Dict = None, adaptor_arguments: Dict = None,
scrapling/engines/pw.py
+3 -2
View File
@@ -1,6 +1,7 @@
import json import json
from scrapling.core._types import Callable, Dict, Optional, Union from scrapling.core._types import (Callable, Dict, Optional,
SelectorWaitStates, Union)
from scrapling.core.utils import log, lru_cache from scrapling.core.utils import log, lru_cache
from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS, from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
NSTBROWSER_DEFAULT_QUERY) NSTBROWSER_DEFAULT_QUERY)
@@ -23,7 +24,7 @@ class PlaywrightEngine:
page_action: Callable = None, page_action: Callable = None,
wait_selector: Optional[str] = None, wait_selector: Optional[str] = None,
locale: Optional[str] = 'en-US', locale: Optional[str] = 'en-US',
wait_selector_state: Optional[str] = 'attached', wait_selector_state: SelectorWaitStates = 'attached',
stealth: Optional[bool] = False, stealth: Optional[bool] = False,
real_chrome: Optional[bool] = False, real_chrome: Optional[bool] = False,
hide_canvas: Optional[bool] = False, hide_canvas: Optional[bool] = False,
scrapling/fetchers.py
+7 -7
View File
@@ -1,5 +1,5 @@
from scrapling.core._types import (Callable, Dict, List, Literal, Optional, from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
Union) SelectorWaitStates, Union)
from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine, from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine,
check_if_engine_usable) check_if_engine_usable)
from scrapling.engines.toolbelt import BaseFetcher, Response from scrapling.engines.toolbelt import BaseFetcher, Response
@@ -176,8 +176,8 @@ class StealthyFetcher(BaseFetcher):
self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False, self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None, block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -234,8 +234,8 @@ class StealthyFetcher(BaseFetcher):
self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False, self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None, block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -308,7 +308,7 @@ class PlayWrightFetcher(BaseFetcher):
def fetch( def fetch(
self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached', page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True, hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
stealth: Optional[bool] = False, real_chrome: Optional[bool] = False, stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
@@ -368,7 +368,7 @@ class PlayWrightFetcher(BaseFetcher):
async def async_fetch( async def async_fetch(
self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached', page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True, hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
stealth: Optional[bool] = False, real_chrome: Optional[bool] = False, stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,