fix(spider robots): removing dead code

scrapling/spiders/engine.py

@@ -96,7 +96,7 @@ class CrawlerEngine:
             return self._domain_delays[domain]
 
         # For domains covered by _prefetch_robots_txt this is a local parser read.
-        # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
+        # Domains discovered mid-crawl (not in start_urls) will fetch here.
         c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
 
         delay = self.spider.download_delay

scrapling/spiders/robotstxt.py

@@ -61,7 +61,6 @@ class RobotsTxtManager:
         """Check if a URL can be fetched according to the domain's robots.txt.
 
         Handles:
-        - User-agent specific rules (e.g., User-agent: SpinarakBot)
         - Wildcard user-agent rules (User-agent: *)
         - Allow/Disallow directives with wildcards (e.g., /*.pdf$)
         - Allow directives that override Disallow (e.g., Allow: /admin/public-docs/)
@@ -80,42 +79,6 @@ class RobotsTxtManager:
         parser = await self._get_parser(url, sid)
         return parser.can_fetch(url, "*")
 
-    async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]:
-        """Get the crawl delay for this crawler.
-
-        Uses the wildcard user-agent (*) to get the general crawl delay
-        that applies to all bots.
-
-        Args:
-            url: Any URL on the domain to check
-            sid: Session ID for fetching robots.txt if not yet cached
-
-        Returns:
-            The crawl delay in seconds, or None if not specified
-        """
-        parser = await self._get_parser(url, sid)
-        delay = parser.crawl_delay("*")
-        return float(delay) if delay is not None else None
-
-    async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]:
-        """Get the request rate for this crawler.
-
-        Uses the wildcard user-agent (*) to get the general request rate
-        that applies to all bots.
-
-        Args:
-            url: Any URL on the domain to check
-            sid: Session ID for fetching robots.txt if not yet cached
-
-        Returns:
-            A tuple of (requests, seconds) if specified, or None if not specified
-        """
-        parser = await self._get_parser(url, sid)
-        rate = parser.request_rate("*")
-        if rate is not None:
-            return (rate.requests, rate.seconds)
-        return None
-
     async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
         """Return both crawl-delay and request-rate in a single parser lookup.
 
@@ -152,18 +115,3 @@ class RobotsTxtManager:
         async with create_task_group() as tg:
             for url in urls:
                 tg.start_soon(self._get_parser, url, sid)
-
-    def clear_cache(self, domain: Optional[str] = None) -> None:
-        """Clear the robots.txt cache.
-
-        Note: the ``sid`` parameter was removed — the cache is now keyed by
-        domain only, so clearing a domain evicts all sessions at once.
-
-        Args:
-            domain: If specified, only clear cache for this domain.
-                    If None, clears the entire cache.
-        """
-        if domain is None:
-            self._cache.clear()
-        else:
-            self._cache.pop(domain, None)

tests/spiders/test_engine.py

@@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt:
         return _fetch, calls
 
     @pytest.mark.asyncio
-    async def test_prefetch_uses_allowed_domains_when_set(self):
+    async def test_prefetch_uses_start_urls(self):
-        fetch_fn, calls = self._make_counting_fetch()
-        spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)
-        engine = _make_engine(spider=spider)
-        engine._robots_manager = RobotsTxtManager(fetch_fn)
-
-        await engine._prefetch_robots_txt()
-
-        fetched_domains = {Request(url).domain for url, _ in calls}
-        assert fetched_domains == {"a.com", "b.com"}
-
-    @pytest.mark.asyncio
-    async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):
         fetch_fn, calls = self._make_counting_fetch()
         spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
         engine = _make_engine(spider=spider)

tests/spiders/test_robotstxt.py

@@ -53,14 +53,6 @@ Request-rate: 1/10
 Disallow: /private/
 """
 
-ROBOTS_WITH_SITEMAP = """\
-User-agent: *
-Disallow:
-
-Sitemap: https://example.com/sitemap.xml
-Sitemap: https://example.com/sitemap2.xml
-"""
-
 ROBOTS_ALLOW_OVERRIDE = """\
 User-agent: *
 Disallow: /secret/
@@ -157,121 +149,79 @@ class TestCanFetch:
 
 
 # ---------------------------------------------------------------------------
-# Tests: get_crawl_delay
+# Tests: get_delay_directives
 # ---------------------------------------------------------------------------
 
 
-class TestGetCrawlDelay:
+class TestGetDelayDirectives:
     @pytest.mark.asyncio
-    async def test_returns_float_when_set(self):
+    async def test_returns_crawl_delay_when_set(self):
         mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
 
-        delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+        c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
 
-        assert delay == 2.0
+        assert c_delay == 2.0
-        assert isinstance(delay, float)
+        assert isinstance(c_delay, float)
+        assert r_rate is None
 
     @pytest.mark.asyncio
-    async def test_returns_none_when_not_set(self):
+    async def test_returns_request_rate_when_set(self):
+        mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
+
+        c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+        assert c_delay is None
+        assert r_rate is not None
+        assert r_rate == (1, 1)
+
+    @pytest.mark.asyncio
+    async def test_returns_both_none_when_not_set(self):
         content = "User-agent: *\nDisallow: /admin/"
         mgr = RobotsTxtManager(make_fetch_fn(content=content))
 
-        assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+        c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+        assert c_delay is None
+        assert r_rate is None
 
     @pytest.mark.asyncio
-    async def test_returns_none_for_empty_robots(self):
+    async def test_returns_both_none_for_empty_robots(self):
         mgr = RobotsTxtManager(make_fetch_fn(content=""))
 
-        assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+        c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+        assert c_delay is None
+        assert r_rate is None
 
     @pytest.mark.asyncio
-    async def test_returns_none_on_fetch_error(self):
+    async def test_returns_both_none_on_fetch_error(self):
         async def failing_fetch(url: str, sid: str) -> MockResponse:
             raise ConnectionError("network failure")
 
         mgr = RobotsTxtManager(failing_fetch)
 
-        assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+        c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+        assert c_delay is None
+        assert r_rate is None
 
     @pytest.mark.asyncio
-    async def test_returns_none_for_non_200_response(self):
+    async def test_fractional_crawl_delay(self):
-        mgr = RobotsTxtManager(make_fetch_fn(status=404))
-
-        assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
-
-    @pytest.mark.asyncio
-    async def test_fractional_delay(self):
         content = "User-agent: *\nCrawl-delay: 0.5"
         mgr = RobotsTxtManager(make_fetch_fn(content=content))
 
-        delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+        c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
 
-        assert delay == 0.5
+        assert c_delay == 0.5
 
     @pytest.mark.asyncio
     async def test_url_path_does_not_affect_result(self):
-        """Any URL on the same domain should return the same delay."""
         mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
 
-        d1 = await mgr.get_crawl_delay("https://example.com/", "s1")
+        r1 = await mgr.get_delay_directives("https://example.com/", "s1")
-        d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1")
+        r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1")
 
-        assert d1 == d2
+        assert r1 == r2
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_request_rate
-# ---------------------------------------------------------------------------
-
-
-class TestGetRequestRate:
-    @pytest.mark.asyncio
-    async def test_returns_tuple_when_set(self):
-        mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
-
-        rate = await mgr.get_request_rate("https://example.com/", "s1")
-
-        assert rate is not None
-        assert isinstance(rate, tuple)
-        assert len(rate) == 2
-
-    @pytest.mark.asyncio
-    async def test_tuple_contains_integers(self):
-        mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
-
-        rate = await mgr.get_request_rate("https://example.com/", "s1")
-
-        assert rate is not None
-        requests, seconds = rate
-        assert isinstance(requests, int)
-        assert isinstance(seconds, int)
-
-    @pytest.mark.asyncio
-    async def test_returns_none_when_not_set(self):
-        mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
-
-        assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
-    @pytest.mark.asyncio
-    async def test_returns_none_for_empty_robots(self):
-        mgr = RobotsTxtManager(make_fetch_fn(content=""))
-
-        assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
-    @pytest.mark.asyncio
-    async def test_returns_none_on_fetch_error(self):
-        async def failing_fetch(url: str, sid: str) -> MockResponse:
-            raise ConnectionError("network failure")
-
-        mgr = RobotsTxtManager(failing_fetch)
-
-        assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
-    @pytest.mark.asyncio
-    async def test_returns_none_for_non_200_response(self):
-        mgr = RobotsTxtManager(make_fetch_fn(status=404))
-
-        assert await mgr.get_request_rate("https://example.com/", "s1") is None
 
 
 # ---------------------------------------------------------------------------
@@ -296,8 +246,7 @@ class TestCachingBehaviour:
         mgr = RobotsTxtManager(fetch_fn)
 
         await mgr.can_fetch("https://example.com/", "s1")
-        await mgr.get_crawl_delay("https://example.com/", "s1")
+        await mgr.get_delay_directives("https://example.com/", "s1")
-        await mgr.get_request_rate("https://example.com/", "s1")
 
         assert len(fetch_fn.calls) == 1
 
@@ -419,9 +368,9 @@ class TestEncoding:
             return MockResponse(status=200, body=body, encoding="latin-1")
 
         mgr = RobotsTxtManager(fetch_fn)
-        delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+        c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
 
-        assert delay == 3.0
+        assert c_delay == 3.0
 
     @pytest.mark.asyncio
     async def test_bytes_body_decoded_correctly(self):
@@ -437,81 +386,6 @@ class TestEncoding:
         assert await mgr.can_fetch("https://example.com/public/", "s1") is True
 
 
-# ---------------------------------------------------------------------------
-# Tests: clear_cache
-# ---------------------------------------------------------------------------
-
-
-class TestClearCache:
-    @pytest.mark.asyncio
-    async def test_clear_all_forces_refetch(self):
-        fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
-        mgr = RobotsTxtManager(fetch_fn)
-
-        await mgr.can_fetch("https://example.com/", "s1")
-        mgr.clear_cache()
-        await mgr.can_fetch("https://example.com/", "s1")
-
-        assert len(fetch_fn.calls) == 2
-
-    @pytest.mark.asyncio
-    async def test_clear_by_domain_only_invalidates_that_domain(self):
-        fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
-        mgr = RobotsTxtManager(fetch_fn)
-
-        await mgr.can_fetch("https://example.com/", "s1")
-        await mgr.can_fetch("https://other.com/", "s1")
-        assert len(fetch_fn.calls) == 2
-
-        mgr.clear_cache(domain="example.com")
-
-        await mgr.can_fetch("https://example.com/", "s1")  # refetched
-        await mgr.can_fetch("https://other.com/", "s1")    # still cached
-
-        assert len(fetch_fn.calls) == 3
-
-    @pytest.mark.asyncio
-    async def test_clear_by_domain_invalidates_all_sessions(self):
-        """Clearing a domain evicts the single shared cache entry for all sessions."""
-        fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
-        mgr = RobotsTxtManager(fetch_fn)
-
-        await mgr.can_fetch("https://example.com/", "s1")
-        assert len(fetch_fn.calls) == 1
-
-        mgr.clear_cache(domain="example.com")
-
-        await mgr.can_fetch("https://example.com/", "s1")  # refetched — cache was cleared
-        await mgr.can_fetch("https://example.com/", "s2")  # hits the newly warm cache, no fetch
-
-        assert len(fetch_fn.calls) == 2
-
-    def test_clear_nonexistent_domain_does_not_raise(self):
-        mgr = RobotsTxtManager(make_fetch_fn())
-        mgr.clear_cache(domain="nevervisited.com")  # should not raise
-
-    def test_clear_empty_cache_does_not_raise(self):
-        mgr = RobotsTxtManager(make_fetch_fn())
-        mgr.clear_cache()  # should not raise
-
-    @pytest.mark.asyncio
-    async def test_clear_all_empties_cache_completely(self):
-        fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
-        mgr = RobotsTxtManager(fetch_fn)
-
-        await mgr.can_fetch("https://a.com/", "s1")
-        await mgr.can_fetch("https://b.com/", "s1")
-        await mgr.can_fetch("https://c.com/", "s1")
-        assert len(fetch_fn.calls) == 3
-
-        mgr.clear_cache()
-
-        await mgr.can_fetch("https://a.com/", "s1")
-        await mgr.can_fetch("https://b.com/", "s1")
-        await mgr.can_fetch("https://c.com/", "s1")
-
-        assert len(fetch_fn.calls) == 6
-
 
 # ---------------------------------------------------------------------------
 # Tests: concurrent access