Making the Response returned from all fetchers same as Adaptor object
but with added attributes
This commit is contained in:
@@ -3,43 +3,29 @@ Functions related to custom types or type checking
|
|||||||
"""
|
"""
|
||||||
import inspect
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass, field
|
|
||||||
|
|
||||||
from scrapling.core.utils import setup_basic_logging
|
from scrapling.core.utils import setup_basic_logging
|
||||||
from scrapling.parser import Adaptor, SQLiteStorageSystem
|
from scrapling.parser import Adaptor, SQLiteStorageSystem
|
||||||
from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
|
from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
class Response(Adaptor):
|
||||||
class Response:
|
|
||||||
"""This class is returned by all engines as a way to unify response type between different libraries."""
|
"""This class is returned by all engines as a way to unify response type between different libraries."""
|
||||||
url: str
|
|
||||||
text: str
|
|
||||||
content: bytes
|
|
||||||
status: int
|
|
||||||
reason: str
|
|
||||||
encoding: str = 'utf-8' # default encoding
|
|
||||||
cookies: Dict = field(default_factory=dict)
|
|
||||||
headers: Dict = field(default_factory=dict)
|
|
||||||
request_headers: Dict = field(default_factory=dict)
|
|
||||||
adaptor_arguments: Dict = field(default_factory=dict)
|
|
||||||
|
|
||||||
@property
|
def __init__(self, url: str, text: str, content: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, adaptor_arguments: Dict, encoding: str = 'utf-8'):
|
||||||
def adaptor(self) -> Union[Adaptor, None]:
|
automatch_domain = adaptor_arguments.pop('automatch_domain', None)
|
||||||
"""Generate Adaptor instance from this response if possible, otherwise return None"""
|
super().__init__(text=text, body=content, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
|
||||||
automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
|
|
||||||
if self.text:
|
|
||||||
# For playwright that will be the response after all JS executed
|
|
||||||
return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
|
|
||||||
elif self.content:
|
|
||||||
# For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
|
|
||||||
# To get response Bytes after the load states
|
|
||||||
# Reference: https://playwright.dev/python/docs/api/class-page
|
|
||||||
return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def __repr__(self):
|
self.status = status
|
||||||
return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
|
self.reason = reason
|
||||||
|
self.cookies = cookies
|
||||||
|
self.headers = headers
|
||||||
|
self.request_headers = request_headers
|
||||||
|
# For back-ward compatibility
|
||||||
|
self.adaptor = self
|
||||||
|
|
||||||
|
# def __repr__(self):
|
||||||
|
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
|
||||||
|
|
||||||
|
|
||||||
class BaseFetcher:
|
class BaseFetcher:
|
||||||
|
|||||||
Reference in New Issue
Block a user