refactor(fetchers): Less duplicated code and better handling for unstable websites

- Websites that never finish loading their requests won't crash the code now if you used `network_idle` with them
- Fixed a typo in `load_dom` in DynamicSession's async_fetch
- Removed dead code
This commit is contained in:
Karim shoair
2025-10-26 05:04:19 +03:00
parent 74fa45daea
commit 59ec6be201
5 changed files with 185 additions and 248 deletions
-12
View File
@@ -209,15 +209,3 @@ class StatusText:
def get(cls, status_code: int) -> str:
"""Get the phrase for a given HTTP status code."""
return cls._phrases.get(status_code, "Unknown Status Code")
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
:return: The name of the variable if found, None otherwise
"""
for scope in [globals(), locals()]:
for name, value in scope.items():
if value is var:
return name
return None
+6 -8
View File
@@ -7,8 +7,9 @@ from platform import system as platform_system
from tldextract import extract
from browserforge.headers import Browser, HeaderGenerator
from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
from scrapling.core._types import Dict, Literal
from scrapling.core._types import Dict, Literal, Tuple
__OS_NAME__ = platform_system()
OSName = Literal["linux", "macos", "windows"]
@@ -29,12 +30,12 @@ def generate_convincing_referer(url: str) -> str:
@lru_cache(1, typed=True)
def get_os_name() -> OSName | None:
def get_os_name() -> OSName | Tuple:
"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
:return: Current OS name or `None` otherwise
"""
match __OS_NAME__:
match __OS_NAME__: # pragma: no cover
case "Linux":
return "linux"
case "Darwin":
@@ -42,7 +43,7 @@ def get_os_name() -> OSName | None:
case "Windows":
return "windows"
case _:
return None
return SUPPORTED_OPERATING_SYSTEMS
def generate_headers(browser_mode: bool = False) -> Dict:
@@ -63,10 +64,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:
Browser(name="edge", min_version=130),
]
)
if os_name:
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
else:
return HeaderGenerator(browser=browsers, device="desktop").generate()
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")