style: Use shorter and more accurate naming for constants
This commit is contained in:
@@ -40,11 +40,7 @@ from scrapling.core._types import (
|
||||
Generator,
|
||||
AsyncGenerator,
|
||||
)
|
||||
from scrapling.engines.constants import (
|
||||
DEFAULT_STEALTH_FLAGS,
|
||||
HARMFUL_DEFAULT_ARGS,
|
||||
DEFAULT_FLAGS,
|
||||
)
|
||||
from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS
|
||||
|
||||
|
||||
class SyncSession:
|
||||
@@ -389,8 +385,8 @@ class BaseSessionMixin:
|
||||
# Dark color scheme bypasses the 'prefersLightColor' check in creepjs
|
||||
self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
|
||||
self._browser_options: Dict[str, Any] = {
|
||||
"args": DEFAULT_FLAGS,
|
||||
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
|
||||
"args": DEFAULT_ARGS,
|
||||
"ignore_default_args": HARMFUL_ARGS,
|
||||
}
|
||||
if "__max_pages" in params:
|
||||
params["max_pages"] = params.pop("__max_pages")
|
||||
@@ -484,7 +480,7 @@ class StealthySessionMixin(BaseSessionMixin):
|
||||
config = cast(StealthConfig, self._config)
|
||||
flags: Tuple[str, ...] = tuple()
|
||||
if not config.cdp_url:
|
||||
flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS
|
||||
flags = DEFAULT_ARGS + STEALTH_ARGS
|
||||
|
||||
if config.block_webrtc:
|
||||
flags += (
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Disable loading these resources for speed
|
||||
DEFAULT_DISABLED_RESOURCES = {
|
||||
EXTRA_RESOURCES = {
|
||||
"font",
|
||||
"image",
|
||||
"media",
|
||||
@@ -12,7 +12,7 @@ DEFAULT_DISABLED_RESOURCES = {
|
||||
"stylesheet",
|
||||
}
|
||||
|
||||
HARMFUL_DEFAULT_ARGS = (
|
||||
HARMFUL_ARGS = (
|
||||
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
|
||||
"--enable-automation",
|
||||
"--disable-popup-blocking",
|
||||
@@ -21,7 +21,7 @@ HARMFUL_DEFAULT_ARGS = (
|
||||
"--disable-extensions",
|
||||
)
|
||||
|
||||
DEFAULT_FLAGS = (
|
||||
DEFAULT_ARGS = (
|
||||
# Speed up chromium browsers by default
|
||||
"--no-pings",
|
||||
"--no-first-run",
|
||||
@@ -36,7 +36,7 @@ DEFAULT_FLAGS = (
|
||||
"--disable-search-engine-choice-screen",
|
||||
)
|
||||
|
||||
DEFAULT_STEALTH_FLAGS = (
|
||||
STEALTH_ARGS = (
|
||||
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
|
||||
# Generally this will make the browser faster and less detectable
|
||||
# "--incognito",
|
||||
|
||||
@@ -12,7 +12,7 @@ from playwright.sync_api import Route
|
||||
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
|
||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
||||
from scrapling.engines.constants import EXTRA_RESOURCES
|
||||
|
||||
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
|
||||
|
||||
@@ -30,7 +30,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
|
||||
:param blocked_domains: Set of domain names to block requests to.
|
||||
:return: A sync route handler function.
|
||||
"""
|
||||
disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
|
||||
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||
domains = blocked_domains or set()
|
||||
|
||||
def handler(route: Route):
|
||||
@@ -57,7 +57,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
|
||||
:param blocked_domains: Set of domain names to block requests to.
|
||||
:return: An async route handler function.
|
||||
"""
|
||||
disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
|
||||
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||
domains = blocked_domains or set()
|
||||
|
||||
async def handler(route: async_Route):
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
from scrapling.engines.constants import (
|
||||
DEFAULT_DISABLED_RESOURCES,
|
||||
DEFAULT_STEALTH_FLAGS,
|
||||
HARMFUL_DEFAULT_ARGS,
|
||||
DEFAULT_FLAGS,
|
||||
)
|
||||
from scrapling.engines.constants import EXTRA_RESOURCES, STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS
|
||||
|
||||
|
||||
class TestConstants:
|
||||
@@ -11,18 +6,18 @@ class TestConstants:
|
||||
|
||||
def test_default_disabled_resources(self):
|
||||
"""Test default disabled resources"""
|
||||
assert "image" in DEFAULT_DISABLED_RESOURCES
|
||||
assert "font" in DEFAULT_DISABLED_RESOURCES
|
||||
assert "stylesheet" in DEFAULT_DISABLED_RESOURCES
|
||||
assert "media" in DEFAULT_DISABLED_RESOURCES
|
||||
assert "image" in EXTRA_RESOURCES
|
||||
assert "font" in EXTRA_RESOURCES
|
||||
assert "stylesheet" in EXTRA_RESOURCES
|
||||
assert "media" in EXTRA_RESOURCES
|
||||
|
||||
def test_harmful_default_args(self):
|
||||
"""Test harmful default arguments"""
|
||||
assert "--enable-automation" in HARMFUL_DEFAULT_ARGS
|
||||
assert "--disable-popup-blocking" in HARMFUL_DEFAULT_ARGS
|
||||
assert "--enable-automation" in HARMFUL_ARGS
|
||||
assert "--disable-popup-blocking" in HARMFUL_ARGS
|
||||
|
||||
def test_flags(self):
|
||||
"""Test default stealth flags"""
|
||||
assert "--no-pings" in DEFAULT_FLAGS
|
||||
# assert "--incognito" in DEFAULT_STEALTH_FLAGS
|
||||
assert "--disable-blink-features=AutomationControlled" in DEFAULT_STEALTH_FLAGS
|
||||
assert "--no-pings" in DEFAULT_ARGS
|
||||
# assert "--incognito" in STEALTH_ARGS
|
||||
assert "--disable-blink-features=AutomationControlled" in STEALTH_ARGS
|
||||
|
||||
@@ -8,7 +8,6 @@ from scrapling.engines.toolbelt.navigation import (
|
||||
create_async_intercept_handler,
|
||||
js_bypass_path,
|
||||
)
|
||||
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
|
||||
from scrapling.engines.toolbelt.fingerprints import (
|
||||
generate_convincing_referer,
|
||||
get_os_name,
|
||||
|
||||
Reference in New Issue
Block a user