refactor: Make all fetchers as an optional dependency group
+ Removing some dead code
This commit is contained in:
@@ -12,20 +12,17 @@ from camoufox.utils import (
|
||||
installed_verstr as camoufox_version,
|
||||
)
|
||||
|
||||
from scrapling.engines.toolbelt import (
|
||||
intercept_route,
|
||||
async_intercept_route,
|
||||
get_os_name,
|
||||
)
|
||||
from ._page import PageInfo, PagePool
|
||||
from ._config_tools import _compiled_stealth_scripts
|
||||
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
|
||||
from ._config_tools import _launch_kwargs, _context_kwargs
|
||||
from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
|
||||
from scrapling.core._types import (
|
||||
Any,
|
||||
Dict,
|
||||
Optional,
|
||||
)
|
||||
from ._page import PageInfo, PagePool
|
||||
from ._config_tools import _compiled_stealth_scripts
|
||||
from ._config_tools import _launch_kwargs, _context_kwargs
|
||||
from scrapling.engines.toolbelt.fingerprints import get_os_name
|
||||
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
|
||||
|
||||
__ff_version_str__ = camoufox_version().split(".", 1)[0]
|
||||
|
||||
|
||||
@@ -25,11 +25,11 @@ from scrapling.core._types import (
|
||||
Callable,
|
||||
SelectorWaitStates,
|
||||
)
|
||||
from scrapling.engines.toolbelt import (
|
||||
from scrapling.engines.toolbelt.convertor import (
|
||||
Response,
|
||||
ResponseFactory,
|
||||
generate_convincing_referer,
|
||||
)
|
||||
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
|
||||
|
||||
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
|
||||
_UNSET = object()
|
||||
|
||||
@@ -6,7 +6,8 @@ from scrapling.engines.constants import (
|
||||
HARMFUL_DEFAULT_ARGS,
|
||||
DEFAULT_FLAGS,
|
||||
)
|
||||
from scrapling.engines.toolbelt import js_bypass_path, generate_headers
|
||||
from scrapling.engines.toolbelt.navigation import js_bypass_path
|
||||
from scrapling.engines.toolbelt.fingerprints import generate_headers
|
||||
|
||||
__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
|
||||
|
||||
|
||||
@@ -26,11 +26,11 @@ from scrapling.core._types import (
|
||||
Callable,
|
||||
SelectorWaitStates,
|
||||
)
|
||||
from scrapling.engines.toolbelt import (
|
||||
from scrapling.engines.toolbelt.convertor import (
|
||||
Response,
|
||||
ResponseFactory,
|
||||
generate_convincing_referer,
|
||||
)
|
||||
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from scrapling.core._types import (
|
||||
List,
|
||||
SelectorWaitStates,
|
||||
)
|
||||
from scrapling.engines.toolbelt import construct_proxy_dict
|
||||
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
|
||||
|
||||
|
||||
class PlaywrightConfig(Struct, kw_only=True, frozen=False):
|
||||
|
||||
Reference in New Issue
Block a user