diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index e869e7f..751d87b 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -102,12 +102,17 @@ class CrawlerEngine: self.stats.offsite_requests_count += 1 log.debug(f"Filtered offsite request to: {result.url}") elif isinstance(result, dict): - self.stats.items_scraped += 1 - self._items.append(result) - if self._item_stream: - await self._item_stream.send(result) - await self.spider.on_scraped_item(result) - log.debug(f"Scraped from {str(response)}\n{result}") + processed_result = await self.spider.on_scraped_item(result) + if processed_result: + self.stats.items_scraped += 1 + log.debug(f"Scraped from {str(response)}\n{processed_result}") + if self._item_stream: + await self._item_stream.send(processed_result) + else: + self._items.append(processed_result) + else: + self.stats.items_dropped += 1 + log.warning(f"Dropped from {str(response)}\n{processed_result}") elif result is not None: log.error(f"Spider must return Request, dict or None, got '{type(result)}' in {request}") except Exception as e: diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py index fc10f1b..7e38a81 100644 --- a/scrapling/spiders/result.py +++ b/scrapling/spiders/result.py @@ -50,6 +50,7 @@ class CrawlStats: offsite_requests_count: int = 0 response_bytes: int = 0 items_scraped: int = 0 + items_dropped: int = 0 start_time: float = 0.0 end_time: float = 0.0 download_delay: float = 0.0 @@ -85,6 +86,7 @@ class CrawlStats: def to_dict(self) -> dict[str, Any]: return { "items_scraped": self.items_scraped, + "items_dropped": self.items_dropped, "elapsed_seconds": round(self.elapsed_seconds, 2), "download_delay": round(self.download_delay, 2), "concurrent_requests": self.concurrent_requests, diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 59ab61c..ac7174f 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -160,9 +160,9 @@ class Spider(ABC): """ self.logger.error(error, exc_info=error) - async def on_scraped_item(self, item: dict[str, Any]) -> None: - """Handle a scraped item. Override or extend for item pipelines.""" - pass + async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None: + """A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently.""" + return item async def is_blocked(self, response: "Response") -> bool: """Check if the response is blocked. Users should override this for custom detection logic."""