From afaf68e7d59e7735e3c6e711f6460dab722a5161 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 02:32:33 +0200 Subject: [PATCH] fix(spider robots): removing dead code --- scrapling/spiders/engine.py | 2 +- scrapling/spiders/robotstxt.py | 52 -------- tests/spiders/test_engine.py | 14 +-- tests/spiders/test_robotstxt.py | 208 +++++++------------------------- 4 files changed, 43 insertions(+), 233 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 7729d62..1924298 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -96,7 +96,7 @@ class CrawlerEngine: return self._domain_delays[domain] # For domains covered by _prefetch_robots_txt this is a local parser read. - # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. + # Domains discovered mid-crawl (not in start_urls) will fetch here. c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid) delay = self.spider.download_delay diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py index 867698b..10dc7ab 100644 --- a/scrapling/spiders/robotstxt.py +++ b/scrapling/spiders/robotstxt.py @@ -61,7 +61,6 @@ class RobotsTxtManager: """Check if a URL can be fetched according to the domain's robots.txt. Handles: - - User-agent specific rules (e.g., User-agent: SpinarakBot) - Wildcard user-agent rules (User-agent: *) - Allow/Disallow directives with wildcards (e.g., /*.pdf$) - Allow directives that override Disallow (e.g., Allow: /admin/public-docs/) @@ -80,42 +79,6 @@ class RobotsTxtManager: parser = await self._get_parser(url, sid) return parser.can_fetch(url, "*") - async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]: - """Get the crawl delay for this crawler. - - Uses the wildcard user-agent (*) to get the general crawl delay - that applies to all bots. - - Args: - url: Any URL on the domain to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - The crawl delay in seconds, or None if not specified - """ - parser = await self._get_parser(url, sid) - delay = parser.crawl_delay("*") - return float(delay) if delay is not None else None - - async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]: - """Get the request rate for this crawler. - - Uses the wildcard user-agent (*) to get the general request rate - that applies to all bots. - - Args: - url: Any URL on the domain to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - A tuple of (requests, seconds) if specified, or None if not specified - """ - parser = await self._get_parser(url, sid) - rate = parser.request_rate("*") - if rate is not None: - return (rate.requests, rate.seconds) - return None - async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: """Return both crawl-delay and request-rate in a single parser lookup. @@ -152,18 +115,3 @@ class RobotsTxtManager: async with create_task_group() as tg: for url in urls: tg.start_soon(self._get_parser, url, sid) - - def clear_cache(self, domain: Optional[str] = None) -> None: - """Clear the robots.txt cache. - - Note: the ``sid`` parameter was removed — the cache is now keyed by - domain only, so clearing a domain evicts all sessions at once. - - Args: - domain: If specified, only clear cache for this domain. - If None, clears the entire cache. - """ - if domain is None: - self._cache.clear() - else: - self._cache.pop(domain, None) diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index e7382f8..74cf875 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt: return _fetch, calls @pytest.mark.asyncio - async def test_prefetch_uses_allowed_domains_when_set(self): - fetch_fn, calls = self._make_counting_fetch() - spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True) - engine = _make_engine(spider=spider) - engine._robots_manager = RobotsTxtManager(fetch_fn) - - await engine._prefetch_robots_txt() - - fetched_domains = {Request(url).domain for url, _ in calls} - assert fetched_domains == {"a.com", "b.com"} - - @pytest.mark.asyncio - async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self): + async def test_prefetch_uses_start_urls(self): fetch_fn, calls = self._make_counting_fetch() spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) engine = _make_engine(spider=spider) diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py index efb8328..c4050c9 100644 --- a/tests/spiders/test_robotstxt.py +++ b/tests/spiders/test_robotstxt.py @@ -53,14 +53,6 @@ Request-rate: 1/10 Disallow: /private/ """ -ROBOTS_WITH_SITEMAP = """\ -User-agent: * -Disallow: - -Sitemap: https://example.com/sitemap.xml -Sitemap: https://example.com/sitemap2.xml -""" - ROBOTS_ALLOW_OVERRIDE = """\ User-agent: * Disallow: /secret/ @@ -157,121 +149,79 @@ class TestCanFetch: # --------------------------------------------------------------------------- -# Tests: get_crawl_delay +# Tests: get_delay_directives # --------------------------------------------------------------------------- -class TestGetCrawlDelay: +class TestGetDelayDirectives: @pytest.mark.asyncio - async def test_returns_float_when_set(self): + async def test_returns_crawl_delay_when_set(self): mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 2.0 - assert isinstance(delay, float) + assert c_delay == 2.0 + assert isinstance(c_delay, float) + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_when_not_set(self): + async def test_returns_request_rate_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is not None + assert r_rate == (1, 1) + + @pytest.mark.asyncio + async def test_returns_both_none_when_not_set(self): content = "User-agent: *\nDisallow: /admin/" mgr = RobotsTxtManager(make_fetch_fn(content=content)) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_for_empty_robots(self): + async def test_returns_both_none_for_empty_robots(self): mgr = RobotsTxtManager(make_fetch_fn(content="")) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_on_fetch_error(self): + async def test_returns_both_none_on_fetch_error(self): async def failing_fetch(url: str, sid: str) -> MockResponse: raise ConnectionError("network failure") mgr = RobotsTxtManager(failing_fetch) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_for_non_200_response(self): - mgr = RobotsTxtManager(make_fetch_fn(status=404)) - - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_fractional_delay(self): + async def test_fractional_crawl_delay(self): content = "User-agent: *\nCrawl-delay: 0.5" mgr = RobotsTxtManager(make_fetch_fn(content=content)) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 0.5 + assert c_delay == 0.5 @pytest.mark.asyncio async def test_url_path_does_not_affect_result(self): - """Any URL on the same domain should return the same delay.""" mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - d1 = await mgr.get_crawl_delay("https://example.com/", "s1") - d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1") + r1 = await mgr.get_delay_directives("https://example.com/", "s1") + r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1") - assert d1 == d2 - - -# --------------------------------------------------------------------------- -# Tests: get_request_rate -# --------------------------------------------------------------------------- - - -class TestGetRequestRate: - @pytest.mark.asyncio - async def test_returns_tuple_when_set(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) - - rate = await mgr.get_request_rate("https://example.com/", "s1") - - assert rate is not None - assert isinstance(rate, tuple) - assert len(rate) == 2 - - @pytest.mark.asyncio - async def test_tuple_contains_integers(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) - - rate = await mgr.get_request_rate("https://example.com/", "s1") - - assert rate is not None - requests, seconds = rate - assert isinstance(requests, int) - assert isinstance(seconds, int) - - @pytest.mark.asyncio - async def test_returns_none_when_not_set(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_for_empty_robots(self): - mgr = RobotsTxtManager(make_fetch_fn(content="")) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_on_fetch_error(self): - async def failing_fetch(url: str, sid: str) -> MockResponse: - raise ConnectionError("network failure") - - mgr = RobotsTxtManager(failing_fetch) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_for_non_200_response(self): - mgr = RobotsTxtManager(make_fetch_fn(status=404)) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None + assert r1 == r2 # --------------------------------------------------------------------------- @@ -296,8 +246,7 @@ class TestCachingBehaviour: mgr = RobotsTxtManager(fetch_fn) await mgr.can_fetch("https://example.com/", "s1") - await mgr.get_crawl_delay("https://example.com/", "s1") - await mgr.get_request_rate("https://example.com/", "s1") + await mgr.get_delay_directives("https://example.com/", "s1") assert len(fetch_fn.calls) == 1 @@ -419,9 +368,9 @@ class TestEncoding: return MockResponse(status=200, body=body, encoding="latin-1") mgr = RobotsTxtManager(fetch_fn) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 3.0 + assert c_delay == 3.0 @pytest.mark.asyncio async def test_bytes_body_decoded_correctly(self): @@ -437,81 +386,6 @@ class TestEncoding: assert await mgr.can_fetch("https://example.com/public/", "s1") is True -# --------------------------------------------------------------------------- -# Tests: clear_cache -# --------------------------------------------------------------------------- - - -class TestClearCache: - @pytest.mark.asyncio - async def test_clear_all_forces_refetch(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - mgr.clear_cache() - await mgr.can_fetch("https://example.com/", "s1") - - assert len(fetch_fn.calls) == 2 - - @pytest.mark.asyncio - async def test_clear_by_domain_only_invalidates_that_domain(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - await mgr.can_fetch("https://other.com/", "s1") - assert len(fetch_fn.calls) == 2 - - mgr.clear_cache(domain="example.com") - - await mgr.can_fetch("https://example.com/", "s1") # refetched - await mgr.can_fetch("https://other.com/", "s1") # still cached - - assert len(fetch_fn.calls) == 3 - - @pytest.mark.asyncio - async def test_clear_by_domain_invalidates_all_sessions(self): - """Clearing a domain evicts the single shared cache entry for all sessions.""" - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - assert len(fetch_fn.calls) == 1 - - mgr.clear_cache(domain="example.com") - - await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared - await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch - - assert len(fetch_fn.calls) == 2 - - def test_clear_nonexistent_domain_does_not_raise(self): - mgr = RobotsTxtManager(make_fetch_fn()) - mgr.clear_cache(domain="nevervisited.com") # should not raise - - def test_clear_empty_cache_does_not_raise(self): - mgr = RobotsTxtManager(make_fetch_fn()) - mgr.clear_cache() # should not raise - - @pytest.mark.asyncio - async def test_clear_all_empties_cache_completely(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://a.com/", "s1") - await mgr.can_fetch("https://b.com/", "s1") - await mgr.can_fetch("https://c.com/", "s1") - assert len(fetch_fn.calls) == 3 - - mgr.clear_cache() - - await mgr.can_fetch("https://a.com/", "s1") - await mgr.can_fetch("https://b.com/", "s1") - await mgr.can_fetch("https://c.com/", "s1") - - assert len(fetch_fn.calls) == 6 - # --------------------------------------------------------------------------- # Tests: concurrent access