diff --git a/scrapling/cli.py b/scrapling/cli.py index 12b96c6..6f9487d 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,8 +2,9 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable +from scrapling.core.utils import log from scrapling.engines.toolbelt.custom import Response -from scrapling.core.utils import log, _CookieParser, _ParseHeaders +from scrapling.core.utils._shell import _CookieParser, _ParseHeaders from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable from orjson import loads as json_loads, JSONDecodeError diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index f9f790f..5ef4458 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -22,10 +22,11 @@ from logging import ( from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ +from scrapling.core.utils import log from scrapling.parser import Selector, Selectors from scrapling.core.custom_types import TextHandler from scrapling.engines.toolbelt.custom import Response -from scrapling.core.utils import log, _ParseHeaders, _CookieParser +from scrapling.core.utils._shell import _ParseHeaders, _CookieParser from scrapling.core._types import ( Optional, Dict, diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index 03c05a2..e832cf0 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -6,7 +6,6 @@ from sqlite3 import connect as db_connect from orjson import dumps, loads from lxml.html import HtmlElement -from tldextract import extract as tld from scrapling.core.utils import _StorageTools, log from scrapling.core._types import Dict, Optional, Any @@ -26,6 +25,8 @@ class StorageSystemMixin(ABC): # pragma: no cover return default_value try: + from tldextract import extract as tld + extracted = tld(self.url) return extracted.top_domain_under_public_suffix or extracted.domain or default_value except AttributeError: diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.py index dc95705..6ae80fe 100644 --- a/scrapling/core/utils/__init__.py +++ b/scrapling/core/utils/__init__.py @@ -7,4 +7,3 @@ from ._utils import ( clean_spaces, html_forbidden, ) -from ._shell import _CookieParser, _ParseHeaders diff --git a/scrapling/parser.py b/scrapling/parser.py index 776dd55..6d0a575 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,8 +1,8 @@ -import re from pathlib import Path from inspect import signature from urllib.parse import urljoin from difflib import SequenceMatcher +from re import Pattern as re_Pattern from lxml.html import HtmlElement, HtmlMixin, HTMLParser from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors @@ -751,7 +751,7 @@ class Selector(SelectorsGeneration): ) attributes.update(arg) - elif isinstance(arg, re.Pattern): + elif isinstance(arg, re_Pattern): patterns.add(arg) elif callable(arg):