Scrapling/scrapling/engines/tools.py

import os
import logging
import inspect
import platform
from scrapling._types import Any, List, Type, Union, Optional
from urllib.parse import urlparse, urlencode

from tldextract import extract
from browserforge.fingerprints import FingerprintGenerator
from browserforge.headers import HeaderGenerator, Browser


def generate_convincing_referer(url):
    """
    Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website

    >>> generate_convincing_referer('https://www.somewebsite.com/blah')
    'https://www.google.com/search?q=somewebsite'

    :param url: The URL you are about to fetch.
    :return:
    """
    website_name = extract(url).domain
    return f'https://www.google.com/search?q={website_name}'


def check_if_engine_usable(engine):
    if isinstance(engine, type):
        raise TypeError("Expected an engine instance, not a class definition of the engine")

    if hasattr(engine, 'fetch'):
        fetch_function = getattr(engine, "fetch")
        if callable(fetch_function):
            if len(inspect.signature(fetch_function).parameters) > 0:
                return engine
            else:
                raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
        else:
            raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
    else:
        raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")


def construct_websocket_url(base_url, query_params):
    # Validate the base URL structure
    try:
        parsed = urlparse(base_url)

        # Check scheme
        if parsed.scheme not in ('ws', 'wss'):
            raise ValueError("URL must use 'ws://' or 'wss://' scheme")

        # Validate hostname and port
        if not parsed.netloc:
            raise ValueError("Invalid hostname")

        # Ensure path starts with /
        path = parsed.path
        if not path.startswith('/'):
            path = '/' + path

        # Reconstruct the base URL with validated parts
        validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"

        # Add query parameters
        if query_params:
            query_string = urlencode(query_params)
            return f"{validated_base}?{query_string}"

        return validated_base

    except Exception as e:
        raise ValueError(f"Invalid WebSocket URL: {str(e)}")


def js_bypass_path(filename):
    current_directory = os.path.dirname(__file__)
    return os.path.join(current_directory, 'bypasses', filename)


def get_os_name():
    # Get the OS name in the same format needed for browserforge
    os_name = platform.system()
    return {
        'Linux': 'linux',
        'Darwin': 'macos',
        'Windows': 'windows',
        # For the future? because why not
        'iOS': 'ios',
    }.get(os_name)


def generate_suitable_fingerprint():
    # This would be for Browserforge playwright injector
    os_name = get_os_name()
    return FingerprintGenerator(
        browser=[Browser(name='chrome', min_version=128)],
        os=os_name,  # None is ignored
        device='desktop'
    ).generate()


def generate_headers(browser_mode=False):
    if browser_mode:
        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
        # So we don't raise any inconsistency red flags while websites fingerprinting us
        os_name = get_os_name()
        return HeaderGenerator(
            browser=[Browser(name='chrome', min_version=128)],
            os=os_name,  # None is ignored
            device='desktop'
        ).generate()
    else:
        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
        browsers = [
            Browser(name='chrome', min_version=120),
            Browser(name='firefox', min_version=120),
            Browser(name='edge', min_version=120),
        ]
        return HeaderGenerator(browser=browsers, device='desktop').generate()


def get_variable_name(var: Any) -> Optional[str]:
    """Get the name of a variable using global and local scopes.
    :param var: The variable to find the name for
    :return: The name of the variable if found, None otherwise
    """
    for scope in [globals(), locals()]:
        for name, value in scope.items():
            if value is var:
                return name
    return None


def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
    """Check if a variable matches the specified type constraints.
    :param variable: The variable to check
    :param valid_types: List of valid types for the variable
    :param default_value: Value to return if type check fails
    :param critical: If True, raises TypeError instead of logging error
    :param param_name: Optional parameter name for error messages
    :return: The original variable if valid, default_value if invalid
    :raise TypeError: If critical=True and type check fails
    """
    # Use provided param_name or try to get it automatically
    var_name = param_name or get_variable_name(variable) or "Unknown"

    # Convert valid_types to a list if None
    valid_types = valid_types or []

    # Handle None value
    if variable is None:
        if type(None) in valid_types:
            return variable
        error_msg = f'Argument "{var_name}" cannot be None'
        if critical:
            raise TypeError(error_msg)
        logging.error(f'[Ignored] {error_msg}')
        return default_value

    # If no valid_types specified and variable has a value, return it
    if not valid_types:
        return variable

    # Check if variable type matches any of the valid types
    if not any(isinstance(variable, t) for t in valid_types):
        type_names = [t.__name__ for t in valid_types]
        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
        if critical:
            raise TypeError(error_msg)
        logging.error(f'[Ignored] {error_msg}')
        return default_value

    return variable