From 23c4181ff239fb306d7031d7266254b170148ea1 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 31 Oct 2024 01:32:18 +0300 Subject: [PATCH] Using the new types file --- scrapling/custom_types.py | 2 +- scrapling/parser.py | 7 +------ scrapling/storage_adaptors.py | 2 +- scrapling/translator.py | 12 +----------- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/scrapling/custom_types.py b/scrapling/custom_types.py index 0c5fb67..5ff3b5f 100644 --- a/scrapling/custom_types.py +++ b/scrapling/custom_types.py @@ -1,9 +1,9 @@ import re from types import MappingProxyType from collections.abc import Mapping -from typing import Dict, List, Union, Pattern from scrapling.utils import _is_iterable, flatten +from scrapling._types import Dict, List, Union, Pattern from orjson import loads, dumps from w3lib.html import replace_entities as _replace_entities diff --git a/scrapling/parser.py b/scrapling/parser.py index a517112..771d08a 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,17 +1,12 @@ import os from difflib import SequenceMatcher -from typing import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator -try: - from typing import SupportsIndex -except ImportError: - # 'SupportsIndex' got added in Python 3.8 - SupportsIndex = None from scrapling.translator import HTMLTranslator from scrapling.mixins import SelectorsGeneration from scrapling.custom_types import TextHandler, AttributesHandler from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden +from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex from lxml import etree, html from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors diff --git a/scrapling/storage_adaptors.py b/scrapling/storage_adaptors.py index ec88925..d9018a3 100644 --- a/scrapling/storage_adaptors.py +++ b/scrapling/storage_adaptors.py @@ -4,8 +4,8 @@ import logging import threading from hashlib import sha256 from abc import ABC, abstractmethod -from typing import Dict, Optional, Union +from scrapling._types import Dict, Optional, Union from scrapling.utils import _StorageTools, cache from lxml import html diff --git a/scrapling/translator.py b/scrapling/translator.py index ec6db86..77679a6 100644 --- a/scrapling/translator.py +++ b/scrapling/translator.py @@ -9,24 +9,14 @@ which will be important in future releases but most importantly... import re from w3lib.html import HTML5_WHITESPACE -from typing import TYPE_CHECKING, Any, Optional -try: - from typing import Protocol -except ImportError: - # Added in Python 3.8 - Protocol = object - from scrapling.utils import cache +from scrapling._types import Any, Optional, Protocol, Self from cssselect.xpath import ExpressionError from cssselect.xpath import XPathExpr as OriginalXPathExpr from cssselect import HTMLTranslator as OriginalHTMLTranslator from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement -if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self - regex = f"[{HTML5_WHITESPACE}]+" replace_html5_whitespaces = re.compile(regex).sub