freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: solve the response log issue so now you get log for every response from Fetchers

Browse Source
This commit is contained in:
Karim shoair
2024-12-19 12:29:34 +02:00
parent 60df72cb23
commit 6652b8ac18
2 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/custom.py
-4
View File
@@ -84,8 +84,6 @@ class ResponseEncoding:
class Response(Adaptor): class Response(Adaptor):
"""This class is returned by all engines as a way to unify response type between different libraries.""" """This class is returned by all engines as a way to unify response type between different libraries."""
_is_response_result_logged = False # Class-level flag, initialized to False
def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict): encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict):
automatch_domain = adaptor_arguments.pop('automatch_domain', None) automatch_domain = adaptor_arguments.pop('automatch_domain', None)
@@ -99,9 +97,7 @@ class Response(Adaptor):
# For back-ward compatibility # For back-ward compatibility
self.adaptor = self self.adaptor = self
# For easier debugging while working from a Python shell # For easier debugging while working from a Python shell
if not Response._is_response_result_logged:
log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})') log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
Response._is_response_result_logged = True
# def __repr__(self): # def __repr__(self):
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>' # return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
scrapling/parser.py
+1 -1
View File
@@ -155,7 +155,7 @@ class Adaptor(SelectorsGeneration):
else: else:
if issubclass(type(element), html.HtmlMixin): if issubclass(type(element), html.HtmlMixin):
return self.__class__( return Adaptor(
root=element, root=element,
text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler
url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled, url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,