From e7cdd39695eb78cc5728ce2d25543468691eb3d0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 30 Jul 2025 01:15:28 +0300 Subject: [PATCH] style: replacing `os` with `Pathlib` and small optimizations --- scrapling/core/shell.py | 8 ++++---- scrapling/engines/_browsers/_validators.py | 8 ++++---- scrapling/engines/toolbelt/navigation.py | 10 ++++++---- scrapling/parser.py | 8 ++++---- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index f11f386..8e95a9a 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -6,7 +6,6 @@ from http import cookies as Cookie from collections import namedtuple from shlex import split as shlex_split from tempfile import mkstemp as make_temp_file -from os import write as os_write, close as os_close from urllib.parse import urlparse, urlunparse, parse_qsl from argparse import ArgumentParser, SUPPRESS from webbrowser import open as open_in_browser @@ -405,9 +404,10 @@ def show_page_in_browser(page: Selector): return try: - fd, fname = make_temp_file(".html") - os_write(fd, page.body.encode("utf-8")) - os_close(fd) + fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html") + with open(fd, "w", encoding="utf-8") as f: + f.write(page.body) + open_in_browser(f"file://{fname}") except IOError as e: log.error(f"Failed to write temporary file for viewing: {e}") diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index e6557ff..2426909 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -1,13 +1,12 @@ from msgspec import Struct, convert, ValidationError from urllib.parse import urlparse -from os.path import exists, isdir +from pathlib import Path from scrapling.core._types import ( Optional, Union, Dict, Callable, - Literal, List, SelectorWaitStates, ) @@ -125,9 +124,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): self.addons = [] else: for addon in self.addons: - if not exists(addon): + addon_path = Path(addon) + if not addon_path.exists(): raise FileNotFoundError(f"Addon's path not found: {addon}") - elif not isdir(addon): + elif not addon_path.is_dir(): raise ValueError( f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}" ) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 95ed535..e03104c 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -2,17 +2,20 @@ Functions related to files and URLs """ -import os +from pathlib import Path +from functools import lru_cache from urllib.parse import urlencode, urlparse from playwright.async_api import Route as async_Route from msgspec import Struct, structs, convert, ValidationError from playwright.sync_api import Route +from scrapling.core.utils import log from scrapling.core._types import Dict, Optional, Union, Tuple -from scrapling.core.utils import log, lru_cache from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES +__BYPASSES_DIR__ = Path(__file__).parent / "bypasses" + class ProxyDict(Struct): server: str @@ -129,5 +132,4 @@ def js_bypass_path(filename: str) -> str: :param filename: The base filename of the JS file. :return: The full path of the JS file. """ - current_directory = os.path.dirname(__file__) - return os.path.join(current_directory, "bypasses", filename) + return str(__BYPASSES_DIR__ / filename) diff --git a/scrapling/parser.py b/scrapling/parser.py index 05d2384..459286a 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import re from inspect import signature from difflib import SequenceMatcher @@ -39,6 +39,8 @@ from scrapling.core.storage import ( from scrapling.core.translator import translator_instance from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log +__DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db") + class Selector(SelectorsGeneration): __slots__ = ( @@ -145,9 +147,7 @@ class Selector(SelectorsGeneration): else: if not storage_args: storage_args = { - "storage_file": os.path.join( - os.path.dirname(__file__), "elements_storage.db" - ), + "storage_file": __DEFAULT_DB_FILE__, "url": url, }