refactor(fetchers): Less duplicated code and better handling for unstable websites
- Websites that never finish loading their requests won't crash the code now if you used `network_idle` with them - Fixed a typo in `load_dom` in DynamicSession's async_fetch - Removed dead code
This commit is contained in:
@@ -209,15 +209,3 @@ class StatusText:
|
||||
def get(cls, status_code: int) -> str:
|
||||
"""Get the phrase for a given HTTP status code."""
|
||||
return cls._phrases.get(status_code, "Unknown Status Code")
|
||||
|
||||
|
||||
def get_variable_name(var: Any) -> Optional[str]:
|
||||
"""Get the name of a variable using global and local scopes.
|
||||
:param var: The variable to find the name for
|
||||
:return: The name of the variable if found, None otherwise
|
||||
"""
|
||||
for scope in [globals(), locals()]:
|
||||
for name, value in scope.items():
|
||||
if value is var:
|
||||
return name
|
||||
return None
|
||||
|
||||
@@ -7,8 +7,9 @@ from platform import system as platform_system
|
||||
|
||||
from tldextract import extract
|
||||
from browserforge.headers import Browser, HeaderGenerator
|
||||
from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
|
||||
|
||||
from scrapling.core._types import Dict, Literal
|
||||
from scrapling.core._types import Dict, Literal, Tuple
|
||||
|
||||
__OS_NAME__ = platform_system()
|
||||
OSName = Literal["linux", "macos", "windows"]
|
||||
@@ -29,12 +30,12 @@ def generate_convincing_referer(url: str) -> str:
|
||||
|
||||
|
||||
@lru_cache(1, typed=True)
|
||||
def get_os_name() -> OSName | None:
|
||||
def get_os_name() -> OSName | Tuple:
|
||||
"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
|
||||
|
||||
:return: Current OS name or `None` otherwise
|
||||
"""
|
||||
match __OS_NAME__:
|
||||
match __OS_NAME__: # pragma: no cover
|
||||
case "Linux":
|
||||
return "linux"
|
||||
case "Darwin":
|
||||
@@ -42,7 +43,7 @@ def get_os_name() -> OSName | None:
|
||||
case "Windows":
|
||||
return "windows"
|
||||
case _:
|
||||
return None
|
||||
return SUPPORTED_OPERATING_SYSTEMS
|
||||
|
||||
|
||||
def generate_headers(browser_mode: bool = False) -> Dict:
|
||||
@@ -63,10 +64,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:
|
||||
Browser(name="edge", min_version=130),
|
||||
]
|
||||
)
|
||||
if os_name:
|
||||
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
|
||||
else:
|
||||
return HeaderGenerator(browser=browsers, device="desktop").generate()
|
||||
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
|
||||
|
||||
|
||||
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
|
||||
|
||||
Reference in New Issue
Block a user