freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(spider robots): removing dead code

Browse Source
This commit is contained in:
Karim shoair
2026-04-05 02:32:33 +02:00
parent ea2dd7866b
commit afaf68e7d5
4 changed files with 43 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/spiders/engine.py
+1 -1
View File
@@ -96,7 +96,7 @@ class CrawlerEngine:
return self._domain_delays[domain]
# For domains covered by _prefetch_robots_txt this is a local parser read.
# Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
# Domains discovered mid-crawl (not in start_urls) will fetch here.
c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
delay = self.spider.download_delay
scrapling/spiders/robotstxt.py
-52
View File
@@ -61,7 +61,6 @@ class RobotsTxtManager:
"""Check if a URL can be fetched according to the domain's robots.txt.
Handles:
- User-agent specific rules (e.g., User-agent: SpinarakBot)
- Wildcard user-agent rules (User-agent: *)
- Allow/Disallow directives with wildcards (e.g., /*.pdf$)
- Allow directives that override Disallow (e.g., Allow: /admin/public-docs/)
@@ -80,42 +79,6 @@ class RobotsTxtManager:
parser = await self._get_parser(url, sid)
return parser.can_fetch(url, "*")
async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]:
"""Get the crawl delay for this crawler.
Uses the wildcard user-agent (*) to get the general crawl delay
that applies to all bots.
Args:
url: Any URL on the domain to check
sid: Session ID for fetching robots.txt if not yet cached
Returns:
The crawl delay in seconds, or None if not specified
"""
parser = await self._get_parser(url, sid)
delay = parser.crawl_delay("*")
return float(delay) if delay is not None else None
async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]:
"""Get the request rate for this crawler.
Uses the wildcard user-agent (*) to get the general request rate
that applies to all bots.
Args:
url: Any URL on the domain to check
sid: Session ID for fetching robots.txt if not yet cached
Returns:
A tuple of (requests, seconds) if specified, or None if not specified
"""
parser = await self._get_parser(url, sid)
rate = parser.request_rate("*")
if rate is not None:
return (rate.requests, rate.seconds)
return None
async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
"""Return both crawl-delay and request-rate in a single parser lookup.
@@ -152,18 +115,3 @@ class RobotsTxtManager:
async with create_task_group() as tg:
for url in urls:
tg.start_soon(self._get_parser, url, sid)
def clear_cache(self, domain: Optional[str] = None) -> None:
"""Clear the robots.txt cache.
Note: the ``sid`` parameter was removed — the cache is now keyed by
domain only, so clearing a domain evicts all sessions at once.
Args:
domain: If specified, only clear cache for this domain.
If None, clears the entire cache.
"""
if domain is None:
self._cache.clear()
else:
self._cache.pop(domain, None)
tests/spiders/test_engine.py
+1 -13
View File
@@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt:
return _fetch, calls
@pytest.mark.asyncio
async def test_prefetch_uses_allowed_domains_when_set(self):
fetch_fn, calls = self._make_counting_fetch()
spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)
engine = _make_engine(spider=spider)
engine._robots_manager = RobotsTxtManager(fetch_fn)
await engine._prefetch_robots_txt()
fetched_domains = {Request(url).domain for url, _ in calls}
assert fetched_domains == {"a.com", "b.com"}
@pytest.mark.asyncio
async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):
async def test_prefetch_uses_start_urls(self):
fetch_fn, calls = self._make_counting_fetch()
spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
engine = _make_engine(spider=spider)
tests/spiders/test_robotstxt.py
+41 -167
View File
@@ -53,14 +53,6 @@ Request-rate: 1/10
Disallow: /private/
"""
ROBOTS_WITH_SITEMAP = """\
User-agent: *
Disallow:
Sitemap: https://example.com/sitemap.xml
Sitemap: https://example.com/sitemap2.xml
"""
ROBOTS_ALLOW_OVERRIDE = """\
User-agent: *
Disallow: /secret/
@@ -157,121 +149,79 @@ class TestCanFetch:
# ---------------------------------------------------------------------------
# Tests: get_crawl_delay
# Tests: get_delay_directives
# ---------------------------------------------------------------------------
class TestGetCrawlDelay:
class TestGetDelayDirectives:
@pytest.mark.asyncio
async def test_returns_float_when_set(self):
async def test_returns_crawl_delay_when_set(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
delay = await mgr.get_crawl_delay("https://example.com/", "s1")
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
assert delay == 2.0
assert isinstance(delay, float)
assert c_delay == 2.0
assert isinstance(c_delay, float)
assert r_rate is None
@pytest.mark.asyncio
async def test_returns_none_when_not_set(self):
async def test_returns_request_rate_when_set(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
assert c_delay is None
assert r_rate is not None
assert r_rate == (1, 1)
@pytest.mark.asyncio
async def test_returns_both_none_when_not_set(self):
content = "User-agent: *\nDisallow: /admin/"
mgr = RobotsTxtManager(make_fetch_fn(content=content))
assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
assert c_delay is None
assert r_rate is None
@pytest.mark.asyncio
async def test_returns_none_for_empty_robots(self):
async def test_returns_both_none_for_empty_robots(self):
mgr = RobotsTxtManager(make_fetch_fn(content=""))
assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
assert c_delay is None
assert r_rate is None
@pytest.mark.asyncio
async def test_returns_none_on_fetch_error(self):
async def test_returns_both_none_on_fetch_error(self):
async def failing_fetch(url: str, sid: str) -> MockResponse:
raise ConnectionError("network failure")
mgr = RobotsTxtManager(failing_fetch)
assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
assert c_delay is None
assert r_rate is None
@pytest.mark.asyncio
async def test_returns_none_for_non_200_response(self):
mgr = RobotsTxtManager(make_fetch_fn(status=404))
assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
@pytest.mark.asyncio
async def test_fractional_delay(self):
async def test_fractional_crawl_delay(self):
content = "User-agent: *\nCrawl-delay: 0.5"
mgr = RobotsTxtManager(make_fetch_fn(content=content))
delay = await mgr.get_crawl_delay("https://example.com/", "s1")
c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
assert delay == 0.5
assert c_delay == 0.5
@pytest.mark.asyncio
async def test_url_path_does_not_affect_result(self):
"""Any URL on the same domain should return the same delay."""
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
d1 = await mgr.get_crawl_delay("https://example.com/", "s1")
d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1")
r1 = await mgr.get_delay_directives("https://example.com/", "s1")
r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1")
assert d1 == d2
# ---------------------------------------------------------------------------
# Tests: get_request_rate
# ---------------------------------------------------------------------------
class TestGetRequestRate:
@pytest.mark.asyncio
async def test_returns_tuple_when_set(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
rate = await mgr.get_request_rate("https://example.com/", "s1")
assert rate is not None
assert isinstance(rate, tuple)
assert len(rate) == 2
@pytest.mark.asyncio
async def test_tuple_contains_integers(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
rate = await mgr.get_request_rate("https://example.com/", "s1")
assert rate is not None
requests, seconds = rate
assert isinstance(requests, int)
assert isinstance(seconds, int)
@pytest.mark.asyncio
async def test_returns_none_when_not_set(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
assert await mgr.get_request_rate("https://example.com/", "s1") is None
@pytest.mark.asyncio
async def test_returns_none_for_empty_robots(self):
mgr = RobotsTxtManager(make_fetch_fn(content=""))
assert await mgr.get_request_rate("https://example.com/", "s1") is None
@pytest.mark.asyncio
async def test_returns_none_on_fetch_error(self):
async def failing_fetch(url: str, sid: str) -> MockResponse:
raise ConnectionError("network failure")
mgr = RobotsTxtManager(failing_fetch)
assert await mgr.get_request_rate("https://example.com/", "s1") is None
@pytest.mark.asyncio
async def test_returns_none_for_non_200_response(self):
mgr = RobotsTxtManager(make_fetch_fn(status=404))
assert await mgr.get_request_rate("https://example.com/", "s1") is None
assert r1 == r2
# ---------------------------------------------------------------------------
@@ -296,8 +246,7 @@ class TestCachingBehaviour:
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://example.com/", "s1")
await mgr.get_crawl_delay("https://example.com/", "s1")
await mgr.get_request_rate("https://example.com/", "s1")
await mgr.get_delay_directives("https://example.com/", "s1")
assert len(fetch_fn.calls) == 1
@@ -419,9 +368,9 @@ class TestEncoding:
return MockResponse(status=200, body=body, encoding="latin-1")
mgr = RobotsTxtManager(fetch_fn)
delay = await mgr.get_crawl_delay("https://example.com/", "s1")
c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
assert delay == 3.0
assert c_delay == 3.0
@pytest.mark.asyncio
async def test_bytes_body_decoded_correctly(self):
@@ -437,81 +386,6 @@ class TestEncoding:
assert await mgr.can_fetch("https://example.com/public/", "s1") is True
# ---------------------------------------------------------------------------
# Tests: clear_cache
# ---------------------------------------------------------------------------
class TestClearCache:
@pytest.mark.asyncio
async def test_clear_all_forces_refetch(self):
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://example.com/", "s1")
mgr.clear_cache()
await mgr.can_fetch("https://example.com/", "s1")
assert len(fetch_fn.calls) == 2
@pytest.mark.asyncio
async def test_clear_by_domain_only_invalidates_that_domain(self):
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://example.com/", "s1")
await mgr.can_fetch("https://other.com/", "s1")
assert len(fetch_fn.calls) == 2
mgr.clear_cache(domain="example.com")
await mgr.can_fetch("https://example.com/", "s1") # refetched
await mgr.can_fetch("https://other.com/", "s1") # still cached
assert len(fetch_fn.calls) == 3
@pytest.mark.asyncio
async def test_clear_by_domain_invalidates_all_sessions(self):
"""Clearing a domain evicts the single shared cache entry for all sessions."""
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://example.com/", "s1")
assert len(fetch_fn.calls) == 1
mgr.clear_cache(domain="example.com")
await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared
await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch
assert len(fetch_fn.calls) == 2
def test_clear_nonexistent_domain_does_not_raise(self):
mgr = RobotsTxtManager(make_fetch_fn())
mgr.clear_cache(domain="nevervisited.com") # should not raise
def test_clear_empty_cache_does_not_raise(self):
mgr = RobotsTxtManager(make_fetch_fn())
mgr.clear_cache() # should not raise
@pytest.mark.asyncio
async def test_clear_all_empties_cache_completely(self):
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://a.com/", "s1")
await mgr.can_fetch("https://b.com/", "s1")
await mgr.can_fetch("https://c.com/", "s1")
assert len(fetch_fn.calls) == 3
mgr.clear_cache()
await mgr.can_fetch("https://a.com/", "s1")
await mgr.can_fetch("https://b.com/", "s1")
await mgr.can_fetch("https://c.com/", "s1")
assert len(fetch_fn.calls) == 6
# ---------------------------------------------------------------------------
# Tests: concurrent access