freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(StealthyFetcher): change additional_arguments to a dictionary argument

Browse Source
This commit is contained in:
Karim shoair
2025-04-08 02:24:43 +02:00
parent c36baf7b4c
commit eb4ca2c6a4
2 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+3 -3
View File
@@ -22,7 +22,7 @@ class CamoufoxEngine:
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
adaptor_arguments: Dict = None, adaptor_arguments: Dict = None,
**additional_arguments: Dict additional_arguments: Dict = None
): ):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation. """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -49,7 +49,7 @@ class CamoufoxEngine:
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param additional_arguments: Any Additional arguments will be passed to Camoufox as additional settings and takes higher priority than Scrapling's settings. :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
""" """
self.headless = headless self.headless = headless
self.block_images = bool(block_images) self.block_images = bool(block_images)
@@ -62,7 +62,7 @@ class CamoufoxEngine:
self.disable_ads = bool(disable_ads) self.disable_ads = bool(disable_ads)
self.geoip = bool(geoip) self.geoip = bool(geoip)
self.extra_headers = extra_headers or {} self.extra_headers = extra_headers or {}
self.additional_arguments = additional_arguments self.additional_arguments = additional_arguments or {}
self.proxy = construct_proxy_dict(proxy) self.proxy = construct_proxy_dict(proxy)
self.addons = addons or [] self.addons = addons or []
self.humanize = humanize self.humanize = humanize
scrapling/fetchers.py
+6 -6
View File
@@ -235,7 +235,7 @@ class StealthyFetcher(BaseFetcher):
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
custom_config: Dict = None, **additional_arguments: Dict custom_config: Dict = None, additional_arguments: Dict = None
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -264,7 +264,7 @@ class StealthyFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Any Additional arguments will be passed to Camoufox as additional settings and takes higher priority than Scrapling's settings. :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
if not custom_config: if not custom_config:
@@ -292,7 +292,7 @@ class StealthyFetcher(BaseFetcher):
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
**additional_arguments additional_arguments=additional_arguments or {}
) )
return engine.fetch(url) return engine.fetch(url)
@@ -303,7 +303,7 @@ class StealthyFetcher(BaseFetcher):
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
custom_config: Dict = None, **additional_arguments: Dict custom_config: Dict = None, additional_arguments: Dict = None
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -332,7 +332,7 @@ class StealthyFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Any Additional arguments will be passed to Camoufox as additional settings and takes higher priority than Scrapling's settings. :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
if not custom_config: if not custom_config:
@@ -360,7 +360,7 @@ class StealthyFetcher(BaseFetcher):
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
**additional_arguments additional_arguments=additional_arguments or {}
) )
return await engine.async_fetch(url) return await engine.async_fetch(url)