Sorting all library imports with isort
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
from .camo import CamoufoxEngine
|
||||
from .static import StaticEngine
|
||||
from .pw import PlaywrightEngine
|
||||
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
|
||||
from .pw import PlaywrightEngine
|
||||
from .static import StaticEngine
|
||||
from .toolbelt import check_if_engine_usable
|
||||
|
||||
__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
|
||||
|
||||
@@ -1,20 +1,16 @@
|
||||
import logging
|
||||
from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
|
||||
|
||||
from scrapling.engines.toolbelt import (
|
||||
Response,
|
||||
do_nothing,
|
||||
StatusText,
|
||||
get_os_name,
|
||||
intercept_route,
|
||||
check_type_validity,
|
||||
construct_proxy_dict,
|
||||
generate_convincing_referer,
|
||||
)
|
||||
|
||||
from camoufox import DefaultAddons
|
||||
from camoufox.sync_api import Camoufox
|
||||
|
||||
from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
|
||||
Union)
|
||||
from scrapling.engines.toolbelt import (Response, StatusText,
|
||||
check_type_validity,
|
||||
construct_proxy_dict, do_nothing,
|
||||
generate_convincing_referer,
|
||||
get_os_name, intercept_route)
|
||||
|
||||
|
||||
class CamoufoxEngine:
|
||||
def __init__(
|
||||
|
||||
+9
-14
@@ -1,20 +1,15 @@
|
||||
import json
|
||||
import logging
|
||||
from scrapling.core._types import Union, Callable, Optional, List, Dict
|
||||
|
||||
from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
|
||||
from scrapling.engines.toolbelt import (
|
||||
Response,
|
||||
do_nothing,
|
||||
StatusText,
|
||||
js_bypass_path,
|
||||
intercept_route,
|
||||
generate_headers,
|
||||
construct_cdp_url,
|
||||
check_type_validity,
|
||||
construct_proxy_dict,
|
||||
generate_convincing_referer,
|
||||
)
|
||||
from scrapling.core._types import Callable, Dict, List, Optional, Union
|
||||
from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
|
||||
NSTBROWSER_DEFAULT_QUERY)
|
||||
from scrapling.engines.toolbelt import (Response, StatusText,
|
||||
check_type_validity, construct_cdp_url,
|
||||
construct_proxy_dict, do_nothing,
|
||||
generate_convincing_referer,
|
||||
generate_headers, intercept_route,
|
||||
js_bypass_path)
|
||||
|
||||
|
||||
class PlaywrightEngine:
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import logging
|
||||
|
||||
from scrapling.core._types import Union, Optional, Dict
|
||||
from .toolbelt import Response, generate_convincing_referer, generate_headers
|
||||
|
||||
import httpx
|
||||
from httpx._models import Response as httpxResponse
|
||||
|
||||
from scrapling.core._types import Dict, Optional, Union
|
||||
|
||||
from .toolbelt import Response, generate_convincing_referer, generate_headers
|
||||
|
||||
|
||||
class StaticEngine:
|
||||
def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
|
||||
|
||||
@@ -1,20 +1,6 @@
|
||||
from .fingerprints import (
|
||||
get_os_name,
|
||||
generate_headers,
|
||||
generate_convincing_referer,
|
||||
)
|
||||
from .custom import (
|
||||
Response,
|
||||
do_nothing,
|
||||
StatusText,
|
||||
BaseFetcher,
|
||||
get_variable_name,
|
||||
check_type_validity,
|
||||
check_if_engine_usable,
|
||||
)
|
||||
from .navigation import (
|
||||
js_bypass_path,
|
||||
intercept_route,
|
||||
construct_cdp_url,
|
||||
construct_proxy_dict,
|
||||
)
|
||||
from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,
|
||||
check_type_validity, do_nothing, get_variable_name)
|
||||
from .fingerprints import (generate_convincing_referer, generate_headers,
|
||||
get_os_name)
|
||||
from .navigation import (construct_cdp_url, construct_proxy_dict,
|
||||
intercept_route, js_bypass_path)
|
||||
|
||||
@@ -5,10 +5,11 @@ import inspect
|
||||
import logging
|
||||
from email.message import Message
|
||||
|
||||
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
|
||||
Type, Union)
|
||||
from scrapling.core.custom_types import MappingProxyType
|
||||
from scrapling.core.utils import cache, setup_basic_logging
|
||||
from scrapling.parser import Adaptor, SQLiteStorageSystem
|
||||
from scrapling.core.utils import setup_basic_logging, cache
|
||||
from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple
|
||||
|
||||
|
||||
class ResponseEncoding:
|
||||
|
||||
@@ -4,12 +4,12 @@ Functions related to generating headers and fingerprints generally
|
||||
|
||||
import platform
|
||||
|
||||
from scrapling.core.utils import cache
|
||||
from scrapling.core._types import Union, Dict
|
||||
|
||||
from browserforge.fingerprints import Fingerprint, FingerprintGenerator
|
||||
from browserforge.headers import Browser, HeaderGenerator
|
||||
from tldextract import extract
|
||||
from browserforge.headers import HeaderGenerator, Browser
|
||||
from browserforge.fingerprints import FingerprintGenerator, Fingerprint
|
||||
|
||||
from scrapling.core._types import Dict, Union
|
||||
from scrapling.core.utils import cache
|
||||
|
||||
|
||||
@cache(None, typed=True)
|
||||
|
||||
@@ -2,16 +2,16 @@
|
||||
Functions related to files and URLs
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from urllib.parse import urlparse, urlencode
|
||||
|
||||
from scrapling.core.utils import cache
|
||||
from scrapling.core._types import Union, Dict, Optional
|
||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
||||
import os
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from playwright.sync_api import Route
|
||||
|
||||
from scrapling.core._types import Dict, Optional, Union
|
||||
from scrapling.core.utils import cache
|
||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
||||
|
||||
|
||||
def intercept_route(route: Route) -> Union[Route, None]:
|
||||
"""This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
|
||||
|
||||
Reference in New Issue
Block a user