style: Use shorter and more accurate naming for constants
This commit is contained in:
@@ -40,11 +40,7 @@ from scrapling.core._types import (
|
|||||||
Generator,
|
Generator,
|
||||||
AsyncGenerator,
|
AsyncGenerator,
|
||||||
)
|
)
|
||||||
from scrapling.engines.constants import (
|
from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS
|
||||||
DEFAULT_STEALTH_FLAGS,
|
|
||||||
HARMFUL_DEFAULT_ARGS,
|
|
||||||
DEFAULT_FLAGS,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SyncSession:
|
class SyncSession:
|
||||||
@@ -389,8 +385,8 @@ class BaseSessionMixin:
|
|||||||
# Dark color scheme bypasses the 'prefersLightColor' check in creepjs
|
# Dark color scheme bypasses the 'prefersLightColor' check in creepjs
|
||||||
self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
|
self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
|
||||||
self._browser_options: Dict[str, Any] = {
|
self._browser_options: Dict[str, Any] = {
|
||||||
"args": DEFAULT_FLAGS,
|
"args": DEFAULT_ARGS,
|
||||||
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
|
"ignore_default_args": HARMFUL_ARGS,
|
||||||
}
|
}
|
||||||
if "__max_pages" in params:
|
if "__max_pages" in params:
|
||||||
params["max_pages"] = params.pop("__max_pages")
|
params["max_pages"] = params.pop("__max_pages")
|
||||||
@@ -484,7 +480,7 @@ class StealthySessionMixin(BaseSessionMixin):
|
|||||||
config = cast(StealthConfig, self._config)
|
config = cast(StealthConfig, self._config)
|
||||||
flags: Tuple[str, ...] = tuple()
|
flags: Tuple[str, ...] = tuple()
|
||||||
if not config.cdp_url:
|
if not config.cdp_url:
|
||||||
flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS
|
flags = DEFAULT_ARGS + STEALTH_ARGS
|
||||||
|
|
||||||
if config.block_webrtc:
|
if config.block_webrtc:
|
||||||
flags += (
|
flags += (
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# Disable loading these resources for speed
|
# Disable loading these resources for speed
|
||||||
DEFAULT_DISABLED_RESOURCES = {
|
EXTRA_RESOURCES = {
|
||||||
"font",
|
"font",
|
||||||
"image",
|
"image",
|
||||||
"media",
|
"media",
|
||||||
@@ -12,7 +12,7 @@ DEFAULT_DISABLED_RESOURCES = {
|
|||||||
"stylesheet",
|
"stylesheet",
|
||||||
}
|
}
|
||||||
|
|
||||||
HARMFUL_DEFAULT_ARGS = (
|
HARMFUL_ARGS = (
|
||||||
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
|
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
|
||||||
"--enable-automation",
|
"--enable-automation",
|
||||||
"--disable-popup-blocking",
|
"--disable-popup-blocking",
|
||||||
@@ -21,7 +21,7 @@ HARMFUL_DEFAULT_ARGS = (
|
|||||||
"--disable-extensions",
|
"--disable-extensions",
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_FLAGS = (
|
DEFAULT_ARGS = (
|
||||||
# Speed up chromium browsers by default
|
# Speed up chromium browsers by default
|
||||||
"--no-pings",
|
"--no-pings",
|
||||||
"--no-first-run",
|
"--no-first-run",
|
||||||
@@ -36,7 +36,7 @@ DEFAULT_FLAGS = (
|
|||||||
"--disable-search-engine-choice-screen",
|
"--disable-search-engine-choice-screen",
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_STEALTH_FLAGS = (
|
STEALTH_ARGS = (
|
||||||
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
|
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
|
||||||
# Generally this will make the browser faster and less detectable
|
# Generally this will make the browser faster and less detectable
|
||||||
# "--incognito",
|
# "--incognito",
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from playwright.sync_api import Route
|
|||||||
|
|
||||||
from scrapling.core.utils import log
|
from scrapling.core.utils import log
|
||||||
from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
|
from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
|
||||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
from scrapling.engines.constants import EXTRA_RESOURCES
|
||||||
|
|
||||||
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
|
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
|
||||||
|
|
||||||
@@ -30,7 +30,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
|
|||||||
:param blocked_domains: Set of domain names to block requests to.
|
:param blocked_domains: Set of domain names to block requests to.
|
||||||
:return: A sync route handler function.
|
:return: A sync route handler function.
|
||||||
"""
|
"""
|
||||||
disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
|
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||||
domains = blocked_domains or set()
|
domains = blocked_domains or set()
|
||||||
|
|
||||||
def handler(route: Route):
|
def handler(route: Route):
|
||||||
@@ -57,7 +57,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
|
|||||||
:param blocked_domains: Set of domain names to block requests to.
|
:param blocked_domains: Set of domain names to block requests to.
|
||||||
:return: An async route handler function.
|
:return: An async route handler function.
|
||||||
"""
|
"""
|
||||||
disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
|
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||||
domains = blocked_domains or set()
|
domains = blocked_domains or set()
|
||||||
|
|
||||||
async def handler(route: async_Route):
|
async def handler(route: async_Route):
|
||||||
|
|||||||
@@ -1,9 +1,4 @@
|
|||||||
from scrapling.engines.constants import (
|
from scrapling.engines.constants import EXTRA_RESOURCES, STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS
|
||||||
DEFAULT_DISABLED_RESOURCES,
|
|
||||||
DEFAULT_STEALTH_FLAGS,
|
|
||||||
HARMFUL_DEFAULT_ARGS,
|
|
||||||
DEFAULT_FLAGS,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestConstants:
|
class TestConstants:
|
||||||
@@ -11,18 +6,18 @@ class TestConstants:
|
|||||||
|
|
||||||
def test_default_disabled_resources(self):
|
def test_default_disabled_resources(self):
|
||||||
"""Test default disabled resources"""
|
"""Test default disabled resources"""
|
||||||
assert "image" in DEFAULT_DISABLED_RESOURCES
|
assert "image" in EXTRA_RESOURCES
|
||||||
assert "font" in DEFAULT_DISABLED_RESOURCES
|
assert "font" in EXTRA_RESOURCES
|
||||||
assert "stylesheet" in DEFAULT_DISABLED_RESOURCES
|
assert "stylesheet" in EXTRA_RESOURCES
|
||||||
assert "media" in DEFAULT_DISABLED_RESOURCES
|
assert "media" in EXTRA_RESOURCES
|
||||||
|
|
||||||
def test_harmful_default_args(self):
|
def test_harmful_default_args(self):
|
||||||
"""Test harmful default arguments"""
|
"""Test harmful default arguments"""
|
||||||
assert "--enable-automation" in HARMFUL_DEFAULT_ARGS
|
assert "--enable-automation" in HARMFUL_ARGS
|
||||||
assert "--disable-popup-blocking" in HARMFUL_DEFAULT_ARGS
|
assert "--disable-popup-blocking" in HARMFUL_ARGS
|
||||||
|
|
||||||
def test_flags(self):
|
def test_flags(self):
|
||||||
"""Test default stealth flags"""
|
"""Test default stealth flags"""
|
||||||
assert "--no-pings" in DEFAULT_FLAGS
|
assert "--no-pings" in DEFAULT_ARGS
|
||||||
# assert "--incognito" in DEFAULT_STEALTH_FLAGS
|
# assert "--incognito" in STEALTH_ARGS
|
||||||
assert "--disable-blink-features=AutomationControlled" in DEFAULT_STEALTH_FLAGS
|
assert "--disable-blink-features=AutomationControlled" in STEALTH_ARGS
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from scrapling.engines.toolbelt.navigation import (
|
|||||||
create_async_intercept_handler,
|
create_async_intercept_handler,
|
||||||
js_bypass_path,
|
js_bypass_path,
|
||||||
)
|
)
|
||||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
|
||||||
from scrapling.engines.toolbelt.fingerprints import (
|
from scrapling.engines.toolbelt.fingerprints import (
|
||||||
generate_convincing_referer,
|
generate_convincing_referer,
|
||||||
get_os_name,
|
get_os_name,
|
||||||
|
|||||||
Reference in New Issue
Block a user