freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: A lot of type hints correction

Browse Source 
Since we are using Py3.10 as minimum version now, we remove Union when possible
This commit is contained in:
Karim shoair
2025-07-30 02:46:57 +03:00
parent ca12a11b7e
commit ae9ccaec79
17 changed files with 179 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+12 -13
View File
@@ -26,10 +26,9 @@ from ._page import PageInfo, PagePool
from ._validators import validate, CamoufoxConfig
from scrapling.core._types import (
Dict,
Optional,
Union,
Callable,
List,
Optional,
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
@@ -84,16 +83,16 @@ class StealthySession:
def __init__(
self,
max_pages: int = 1,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
@@ -101,7 +100,7 @@ class StealthySession:
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
@@ -461,16 +460,16 @@ class AsyncStealthySession(StealthySession):
def __init__(
self,
max_pages: int = 1,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
@@ -478,7 +477,7 @@ class AsyncStealthySession(StealthySession):
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
scrapling/engines/_browsers/_controllers.py
+7 -8
View File
@@ -28,9 +28,8 @@ from ._validators import validate, PlaywrightConfig
from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
from scrapling.core._types import (
Dict,
Optional,
Union,
List,
Optional,
Callable,
SelectorWaitStates,
)
@@ -87,14 +86,14 @@ class DynamicSession:
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
@@ -404,14 +403,14 @@ class AsyncDynamicSession(DynamicSession):
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
scrapling/engines/_browsers/_page.py
+3 -3
View File
@@ -4,7 +4,7 @@ from dataclasses import dataclass
from playwright.sync_api import Page as SyncPage
from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, Union, List, Literal
from scrapling.core._types import Optional, List, Literal
PageState = Literal["ready", "busy", "error"] # States that a page can be in
@@ -14,7 +14,7 @@ class PageInfo:
"""Information about the page and its current state"""
__slots__ = ("page", "state", "url")
page: Union[SyncPage, AsyncPage]
page: SyncPage | AsyncPage
state: PageState
url: Optional[str]
@@ -52,7 +52,7 @@ class PagePool:
self.pages: List[PageInfo] = []
self._lock = RLock()
def add_page(self, page: Union[SyncPage, AsyncPage]) -> PageInfo:
def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:
"""Add a new page to the pool"""
with self._lock:
if len(self.pages) >= self.max_pages:
scrapling/engines/_browsers/_validators.py
+7 -8
View File
@@ -4,7 +4,6 @@ from pathlib import Path
from scrapling.core._types import (
Optional,
Union,
Dict,
Callable,
List,
@@ -24,15 +23,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
disable_webgl: bool = False
real_chrome: bool = False
stealth: bool = False
wait: Union[int, float] = 0
wait: int | float = 0
page_action: Optional[Callable] = None
proxy: Optional[Union[str, Dict[str, str]]] = (
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: Union[int, float] = 30000
timeout: int | float = 30000
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
@@ -87,10 +86,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
block_webrtc: bool = False
allow_webgl: bool = True
network_idle: bool = False
humanize: Union[bool, float] = True
humanize: bool | float = True
solve_cloudflare: bool = False
wait: Union[int, float] = 0
timeout: Union[int, float] = 30000
wait: int | float = 0
timeout: int | float = 30000
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
@@ -98,7 +97,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
cookies: Optional[List[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[Union[str, Dict[str, str]]] = (
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
os_randomize: bool = False