style(spiders robots feat): Adjustments for maintainability

2026-04-04 21:06:35 +02:00
parent acb49b178c
commit 854daac794
2 changed files with 5 additions and 4 deletions
@@ -116,7 +116,7 @@ class RobotsTxtManager:
            return (rate.requests, rate.seconds)
        return None

-    async def _get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
+    async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
        """Return both crawl-delay and request-rate in a single parser lookup.

        Args: