From 3466bcab2c029abefecfc599f61e30351307b51d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 12 Jan 2026 15:58:33 +0200 Subject: [PATCH] feat: add hook to run after collecting an item --- scrapling/spiders/engine.py | 9 +++------ scrapling/spiders/spider.py | 4 ++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index bd50c41..2b12683 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -102,18 +102,15 @@ class CrawlerEngine: self.stats.offsite_requests_count += 1 log.debug(f"Filtered offsite request to: {result.url}") elif isinstance(result, dict): - await self._handle_item(result) + self.stats.items_scraped += 1 + self._items.append(result) if self._item_stream: await self._item_stream.send(result) + await self.spider.on_scraped_item(result) log.debug(f"Scraped from {str(response)}\n{result}") except Exception as e: await self.spider.on_error(request, e) - async def _handle_item(self, item: dict[str, Any]) -> None: - """Handle a scraped item. Override or extend for item pipelines.""" - self.stats.items_scraped += 1 - self._items.append(item) - async def _task_wrapper(self, request: Request) -> None: """Wrapper to track active task count.""" try: diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index eb5987f..bc7874c 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -158,6 +158,10 @@ class Spider(ABC): """ self.logger.error(error, exc_info=error) + async def on_scraped_item(self, item: dict[str, Any]) -> None: + """Handle a scraped item. Override or extend for item pipelines.""" + pass + async def is_blocked(self, response: "Response") -> bool: """Check if the response is blocked. Users should override this for custom detection logic.""" if response.status in BLOCKED_CODES: