feat(spiders): Add logic to detect blocked requests and retry them

This commit is contained in:
Karim shoair
2026-01-11 21:56:57 +02:00
parent 774364b6d3
commit 736873a7c6
3 changed files with 38 additions and 5 deletions
+15
View File
@@ -13,6 +13,7 @@ class Request:
priority: int = 0,
dont_filter: bool = False,
meta: dict[str, Any] | None = None,
_retry_count: int = 0,
**kwargs: Any,
) -> None:
self.url: str = url
@@ -21,8 +22,22 @@ class Request:
self.priority: int = priority
self.dont_filter: bool = dont_filter
self.meta: dict[str, Any] = meta if meta else {}
self._retry_count: int = _retry_count
self._session_kwargs = kwargs if kwargs else {}
def copy(self) -> "Request":
"""Create a copy of this request."""
return Request(
url=self.url,
sid=self.sid,
callback=self.callback,
priority=self.priority,
dont_filter=self.dont_filter,
meta=self.meta.copy(),
_retry_count=self._retry_count,
**self._session_kwargs,
)
@property
def domain(self) -> str:
return urlparse(self.url).netloc