freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(security): default follow_redirects to "safe" for SSRF protection

Browse Source 
curl_cffi v0.15.0 introduced CurlFollow.SAFE, which follows redirects but rejects those targeting internal/private IPs (loopback, private networks, link-local). This is now the default for all HTTP fetchers, the MCP server, and the shell curl converter.

Added FollowRedirects type alias supporting all curl_cffi redirect
modes: bool, "safe", "all", "obeycode", "firstonly".
This commit is contained in:
Karim shoair
2026-04-05 18:41:50 +02:00
parent 065bf1c0e8
commit e7f9adb40a
6 changed files with 23 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_shell_signatures.py
+2 -1
View File
@@ -8,6 +8,7 @@ from scrapling.core._types import (
Optional,
SetCookieParam,
SelectorWaitStates,
FollowRedirects,
)
# Parameter definitions for shell function signatures (defined once at module level)
@@ -26,7 +27,7 @@ _REQUESTS_PARAMS = {
"headers": Any,
"retries": Optional[int],
"retry_delay": Optional[int],
"follow_redirects": Optional[bool],
"follow_redirects": Optional[FollowRedirects],
"max_redirects": Optional[int],
"verify": Optional[bool],
"cert": Optional[str | Tuple[str, str]],
scrapling/core/_types.py
+1
View File
@@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]]
# Copied from `playwright._impl._api_structures.SetCookieParam`
scrapling/core/ai.py
+5 -4
View File
@@ -27,6 +27,7 @@ from scrapling.core._types import (
SetCookieParam,
extraction_types,
SelectorWaitStates,
FollowRedirects,
)
SessionType = Literal["dynamic", "stealthy"]
@@ -262,7 +263,7 @@ class ScraplingMCPServer:
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
@@ -289,7 +290,7 @@ class ScraplingMCPServer:
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -335,7 +336,7 @@ class ScraplingMCPServer:
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
@@ -362,7 +363,7 @@ class ScraplingMCPServer:
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
scrapling/core/shell.py
+1 -1
View File
@@ -294,7 +294,7 @@ class CurlParser:
headers=headers,
cookies=cookies,
proxy=proxies,
follow_redirects=True, # Scrapling default is True
follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs
)
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
scrapling/engines/_browsers/_types.py
+2 -1
View File
@@ -19,6 +19,7 @@ from scrapling.core._types import (
TypeAlias,
SetCookieParam,
SelectorWaitStates,
FollowRedirects,
)
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):
headers: Optional[Mapping[str, Optional[str]]]
retries: Optional[int]
retry_delay: Optional[int]
follow_redirects: Optional[bool]
follow_redirects: Optional[FollowRedirects]
max_redirects: Optional[int]
verify: Optional[bool]
cert: Optional[str | Tuple[str, str]]
scrapling/engines/static.py
+12 -11
View File
@@ -20,6 +20,7 @@ from scrapling.core._types import (
Optional,
Awaitable,
SUPPORTED_HTTP_METHODS,
FollowRedirects,
)
from .toolbelt.custom import Response
@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):
self._default_headers = kwargs.get("headers") or {}
self._default_retries = kwargs.get("retries", 3)
self._default_retry_delay = kwargs.get("retry_delay", 1)
self._default_follow_redirects = kwargs.get("follow_redirects", True)
self._default_follow_redirects = kwargs.get("follow_redirects", "safe")
self._default_max_redirects = kwargs.get("max_redirects", 30)
self._default_verify = kwargs.get("verify", True)
self._default_cert = kwargs.get("cert") or None
@@ -285,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -317,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -349,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -381,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -502,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -534,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -566,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -598,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- follow_redirects: Whether to follow redirects. Defaults to True.
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -663,7 +664,7 @@ class FetcherSession:
headers: Optional[Dict[str, str]] = None,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
follow_redirects: bool = True,
follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
verify: bool = True,
cert: Optional[str | Tuple[str, str]] = None,
@@ -682,7 +683,7 @@ class FetcherSession:
:param headers: Headers to include in the session with every request.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.