diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 1abeb76..463a290 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -207,6 +207,7 @@ class CrawlerEngine: self._items.clear() self.paused = False self._pause_requested = False + self._force_stop = False self.stats = CrawlStats(start_time=anyio.current_time()) # Check for existing checkpoint diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index bde8f78..649f4d3 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -211,7 +211,9 @@ class Spider(ABC): manager.add("default", FetcherSession()) def pause(self): - """Pause the crawling process if checkpoint system is enabled.""" + """Pause the crawling process. Requires crawldir to be set for checkpoint system.""" + if not self.crawldir: + raise RuntimeError("Cannot pause without crawldir - checkpoint system not enabled") if self._engine: self._engine.request_pause() else: