fix(spiders): corrections and removal of dead code
This commit is contained in:
@@ -32,7 +32,7 @@ class CheckpointManager:
|
||||
if not isinstance(interval, (int, float)):
|
||||
raise TypeError("Checkpoints interval must be integer or float.")
|
||||
else:
|
||||
if interval < 0:
|
||||
if interval <= 0:
|
||||
raise ValueError("Checkpoints interval must be greater than 0.")
|
||||
|
||||
async def has_checkpoint(self) -> bool:
|
||||
|
||||
@@ -85,8 +85,6 @@ class Request:
|
||||
body = post_data.getvalue()
|
||||
elif isinstance(post_data, bytes):
|
||||
body = post_data
|
||||
elif post_data is None:
|
||||
body = b""
|
||||
else:
|
||||
post_data = self._session_kwargs.get("json", {})
|
||||
body = orjson.dumps(post_data) if post_data else b""
|
||||
|
||||
@@ -15,7 +15,6 @@ class ItemList(list):
|
||||
|
||||
:param path: Path to the output file
|
||||
:param indent: Pretty-print with 2-space indentation (slightly slower)
|
||||
:return: Number of items written
|
||||
"""
|
||||
options = orjson.OPT_SERIALIZE_NUMPY
|
||||
if indent:
|
||||
|
||||
@@ -79,9 +79,9 @@ class Spider(ABC):
|
||||
max_blocked_retries: int = 3
|
||||
|
||||
# Fingerprint adjustments
|
||||
fp_include_kwargs = False
|
||||
fp_keep_fragments = False
|
||||
fp_include_headers = False
|
||||
fp_include_kwargs: bool = False
|
||||
fp_keep_fragments: bool = False
|
||||
fp_include_headers: bool = False
|
||||
|
||||
# Logging settings
|
||||
logging_level: int = logging.DEBUG
|
||||
|
||||
Reference in New Issue
Block a user