refactor: Make all fetchers as an optional dependency group

+ Removing some dead code
This commit is contained in:
Karim shoair
2025-09-13 16:07:48 +03:00
parent d7e3deae2a
commit 13d7e70cb7
20 changed files with 142 additions and 174 deletions
+6 -9
View File
@@ -12,20 +12,17 @@ from camoufox.utils import (
installed_verstr as camoufox_version,
)
from scrapling.engines.toolbelt import (
intercept_route,
async_intercept_route,
get_os_name,
)
from ._page import PageInfo, PagePool
from ._config_tools import _compiled_stealth_scripts
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
from ._config_tools import _launch_kwargs, _context_kwargs
from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
from scrapling.core._types import (
Any,
Dict,
Optional,
)
from ._page import PageInfo, PagePool
from ._config_tools import _compiled_stealth_scripts
from ._config_tools import _launch_kwargs, _context_kwargs
from scrapling.engines.toolbelt.fingerprints import get_os_name
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
__ff_version_str__ = camoufox_version().split(".", 1)[0]
+2 -2
View File
@@ -25,11 +25,11 @@ from scrapling.core._types import (
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
from scrapling.engines.toolbelt.convertor import (
Response,
ResponseFactory,
generate_convincing_referer,
)
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
_UNSET = object()
+2 -1
View File
@@ -6,7 +6,8 @@ from scrapling.engines.constants import (
HARMFUL_DEFAULT_ARGS,
DEFAULT_FLAGS,
)
from scrapling.engines.toolbelt import js_bypass_path, generate_headers
from scrapling.engines.toolbelt.navigation import js_bypass_path
from scrapling.engines.toolbelt.fingerprints import generate_headers
__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
+2 -2
View File
@@ -26,11 +26,11 @@ from scrapling.core._types import (
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
from scrapling.engines.toolbelt.convertor import (
Response,
ResponseFactory,
generate_convincing_referer,
)
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
_UNSET = object()
+1 -1
View File
@@ -9,7 +9,7 @@ from scrapling.core._types import (
List,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import construct_proxy_dict
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
class PlaywrightConfig(Struct, kw_only=True, frozen=False):