freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(requests): Fix multiple configuration issues

Browse Source
This commit is contained in:
Karim shoair
2025-10-06 06:53:09 +03:00
parent 390916ba45
commit 9cae97b177
1 changed files with 4 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+4 -7
View File
@@ -94,7 +94,8 @@ class _ConfigurationLogic(ABC):
def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments.""" """Merge request-specific arguments with default session arguments."""
url = method_kwargs.pop("url") url = method_kwargs.pop("url")
impersonate = self._get_with_precedence(method_kwargs.get("impersonate"), self._default_impersonate) impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)
http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3)
final_args = { final_args = {
"url": url, "url": url,
# Curl automatically generates the suitable browser headers when you use `impersonate` # Curl automatically generates the suitable browser headers when you use `impersonate`
@@ -125,7 +126,7 @@ class _ConfigurationLogic(ABC):
) )
}, # Add any remaining parameters (after all known ones are popped) }, # Add any remaining parameters (after all known ones are popped)
} }
if self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3): # pragma: no cover if http3_enabled: # pragma: no cover
final_args["http_version"] = CurlHttpVersion.V3ONLY final_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate: if impersonate:
log.warning( log.warning(
@@ -141,7 +142,7 @@ class _ConfigurationLogic(ABC):
3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain. 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain.
""" """
# Merge session headers with request headers, request takes precedence (if it was set) # Merge session headers with request headers, request takes precedence (if it was set)
final_headers = {**self._default_headers, **(headers if headers is not _UNSET else {})} final_headers = {**self._default_headers, **(headers if headers and headers is not _UNSET else {})}
headers_keys = {k.lower() for k in final_headers} headers_keys = {k.lower() for k in final_headers}
if stealth: if stealth:
if "referer" not in headers_keys: if "referer" not in headers_keys:
@@ -252,8 +253,6 @@ class _SyncSessionLogic(_ConfigurationLogic):
try: try:
response = session.request(method, **request_args) response = session.request(method, **request_args)
result = ResponseFactory.from_http_request(response, selector_config) result = ResponseFactory.from_http_request(response, selector_config)
if one_off_request:
session.close()
return result return result
except CurlError as e: # pragma: no cover except CurlError as e: # pragma: no cover
if attempt < max_retries - 1: if attempt < max_retries - 1:
@@ -667,8 +666,6 @@ class _ASyncSessionLogic(_ConfigurationLogic):
try: try:
response = await session.request(method, **request_args) response = await session.request(method, **request_args)
result = ResponseFactory.from_http_request(response, selector_config) result = ResponseFactory.from_http_request(response, selector_config)
if one_off_request:
await session.close()
return result return result
except CurlError as e: # pragma: no cover except CurlError as e: # pragma: no cover
if attempt < max_retries - 1: if attempt < max_retries - 1: