diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index d9a42fc..9778318 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -8,6 +8,7 @@ from scrapling.core._types import ( Optional, SetCookieParam, SelectorWaitStates, + FollowRedirects, ) # Parameter definitions for shell function signatures (defined once at module level) @@ -26,7 +27,7 @@ _REQUESTS_PARAMS = { "headers": Any, "retries": Optional[int], "retry_delay": Optional[int], - "follow_redirects": Optional[bool], + "follow_redirects": Optional[FollowRedirects], "max_redirects": Optional[int], "verify": Optional[bool], "cert": Optional[str | Tuple[str, str]], diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 25ddb5b..fc8a075 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"] extraction_types = Literal["text", "html", "markdown"] StrOrBytes = Union[str, bytes] +FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]] # Copied from `playwright._impl._api_structures.SetCookieParam` diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 208f3e7..9b33821 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -27,6 +27,7 @@ from scrapling.core._types import ( SetCookieParam, extraction_types, SelectorWaitStates, + FollowRedirects, ) SessionType = Literal["dynamic", "stealthy"] @@ -262,7 +263,7 @@ class ScraplingMCPServer: headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, @@ -289,7 +290,7 @@ class ScraplingMCPServer: :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -335,7 +336,7 @@ class ScraplingMCPServer: headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, @@ -362,7 +363,7 @@ class ScraplingMCPServer: :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 14a25fe..cb805c6 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -294,7 +294,7 @@ class CurlParser: headers=headers, cookies=cookies, proxy=proxies, - follow_redirects=True, # Scrapling default is True + follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs ) def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]: diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 6ab40b9..2741407 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -19,6 +19,7 @@ from scrapling.core._types import ( TypeAlias, SetCookieParam, SelectorWaitStates, + FollowRedirects, ) from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator @@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False): headers: Optional[Mapping[str, Optional[str]]] retries: Optional[int] retry_delay: Optional[int] - follow_redirects: Optional[bool] + follow_redirects: Optional[FollowRedirects] max_redirects: Optional[int] verify: Optional[bool] cert: Optional[str | Tuple[str, str]] diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index a962ccf..fae9f83 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -20,6 +20,7 @@ from scrapling.core._types import ( Optional, Awaitable, SUPPORTED_HTTP_METHODS, + FollowRedirects, ) from .toolbelt.custom import Response @@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC): self._default_headers = kwargs.get("headers") or {} self._default_retries = kwargs.get("retries", 3) self._default_retry_delay = kwargs.get("retry_delay", 1) - self._default_follow_redirects = kwargs.get("follow_redirects", True) + self._default_follow_redirects = kwargs.get("follow_redirects", "safe") self._default_max_redirects = kwargs.get("max_redirects", 30) self._default_verify = kwargs.get("verify", True) self._default_cert = kwargs.get("cert") or None @@ -285,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -317,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -349,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -381,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -502,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -534,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -566,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -598,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -663,7 +664,7 @@ class FetcherSession: headers: Optional[Dict[str, str]] = None, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, verify: bool = True, cert: Optional[str | Tuple[str, str]] = None, @@ -682,7 +683,7 @@ class FetcherSession: :param headers: Headers to include in the session with every request. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param verify: Whether to verify HTTPS certificates. Defaults to True. :param cert: Tuple of (cert, key) filenames for the client certificate.