fix(spiders): corrections and removal of dead code
This commit is contained in:
@@ -32,7 +32,7 @@ class CheckpointManager:
|
|||||||
if not isinstance(interval, (int, float)):
|
if not isinstance(interval, (int, float)):
|
||||||
raise TypeError("Checkpoints interval must be integer or float.")
|
raise TypeError("Checkpoints interval must be integer or float.")
|
||||||
else:
|
else:
|
||||||
if interval < 0:
|
if interval <= 0:
|
||||||
raise ValueError("Checkpoints interval must be greater than 0.")
|
raise ValueError("Checkpoints interval must be greater than 0.")
|
||||||
|
|
||||||
async def has_checkpoint(self) -> bool:
|
async def has_checkpoint(self) -> bool:
|
||||||
|
|||||||
@@ -85,8 +85,6 @@ class Request:
|
|||||||
body = post_data.getvalue()
|
body = post_data.getvalue()
|
||||||
elif isinstance(post_data, bytes):
|
elif isinstance(post_data, bytes):
|
||||||
body = post_data
|
body = post_data
|
||||||
elif post_data is None:
|
|
||||||
body = b""
|
|
||||||
else:
|
else:
|
||||||
post_data = self._session_kwargs.get("json", {})
|
post_data = self._session_kwargs.get("json", {})
|
||||||
body = orjson.dumps(post_data) if post_data else b""
|
body = orjson.dumps(post_data) if post_data else b""
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ class ItemList(list):
|
|||||||
|
|
||||||
:param path: Path to the output file
|
:param path: Path to the output file
|
||||||
:param indent: Pretty-print with 2-space indentation (slightly slower)
|
:param indent: Pretty-print with 2-space indentation (slightly slower)
|
||||||
:return: Number of items written
|
|
||||||
"""
|
"""
|
||||||
options = orjson.OPT_SERIALIZE_NUMPY
|
options = orjson.OPT_SERIALIZE_NUMPY
|
||||||
if indent:
|
if indent:
|
||||||
|
|||||||
@@ -79,9 +79,9 @@ class Spider(ABC):
|
|||||||
max_blocked_retries: int = 3
|
max_blocked_retries: int = 3
|
||||||
|
|
||||||
# Fingerprint adjustments
|
# Fingerprint adjustments
|
||||||
fp_include_kwargs = False
|
fp_include_kwargs: bool = False
|
||||||
fp_keep_fragments = False
|
fp_keep_fragments: bool = False
|
||||||
fp_include_headers = False
|
fp_include_headers: bool = False
|
||||||
|
|
||||||
# Logging settings
|
# Logging settings
|
||||||
logging_level: int = logging.DEBUG
|
logging_level: int = logging.DEBUG
|
||||||
|
|||||||
Reference in New Issue
Block a user