From 97fecc9d5fe6f428a9be5268f14f3e290b01d04a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 29 Nov 2024 21:14:52 +0200 Subject: [PATCH 01/13] Style adjustments by flake8 --- .flake8 | 4 ++-- scrapling/core/utils.py | 14 +++++++------- scrapling/engines/toolbelt/navigation.py | 2 +- setup.py | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.flake8 b/.flake8 index 5a89b58..fae58af 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] -ignore = E501 # line too long -exclude = .git,__pycache__,docs,.github,build,dist \ No newline at end of file +ignore = E501, F401 +exclude = .git,.venv,__pycache__,docs,.github,build,dist,tests,benchmarks.py \ No newline at end of file diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index 748020b..3cf887c 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -11,12 +11,12 @@ from lxml import html html_forbidden = {html.HtmlComment, } logging.basicConfig( - level=logging.ERROR, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler() - ] - ) + level=logging.ERROR, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler() + ] +) def is_jsonable(content: Union[bytes, str]) -> bool: @@ -94,7 +94,7 @@ class _StorageTools: parent = element.getparent() return tuple( (element.tag,) if parent is None else ( - cls._get_element_path(parent) + (element.tag,) + cls._get_element_path(parent) + (element.tag,) ) ) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 363f233..faf4402 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -43,7 +43,7 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict } except ValueError: # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc... - raise TypeError(f'The proxy argument\'s string is in invalid format!') + raise TypeError('The proxy argument\'s string is in invalid format!') elif isinstance(proxy_string, dict): valid_keys = ('server', 'username', 'password', ) diff --git a/setup.py b/setup.py index 91aa89c..c360b7a 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,9 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", version="0.2.7", - description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It - simplifies the process of extracting data from websites, even when they undergo structural changes, and offers - impressive speed improvements over many popular scraping tools.""", + description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It + simplifies the process of extracting data from websites, even when they undergo structural changes, and offers + impressive speed improvements over many popular scraping tools.""", long_description=long_description, long_description_content_type="text/markdown", author="Karim Shoair", From 1cf7a1d20534f019944315837625a781586a7dab Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 29 Nov 2024 21:19:39 +0200 Subject: [PATCH 02/13] Make Github tests workflow work on main and dev branches only The docs branch doesn't need tests --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a56cd39..473435b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,9 @@ name: Tests -on: [push] +on: + push: + branches: + - main + - dev concurrency: group: ${{github.workflow}}-${{ github.ref }} From 82f8e4a07332c0436c69806727c219bfa4d376e5 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 11:23:07 +0200 Subject: [PATCH 03/13] Fixing import typo, thanks to #20 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca23a6f..9f7a86e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Dealing with failing web scrapers due to anti-bot protections or website changes Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. For both beginners and experts, Scrapling provides powerful features while maintaining simplicity. ```python ->> from scrapling.default import Fetcher, StealthyFetcher, PlayWrightFetcher +>> from scrapling.defaults import Fetcher, StealthyFetcher, PlayWrightFetcher # Fetch websites' source under the radar! >> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) >> print(page.status) @@ -223,7 +223,7 @@ All of them can take these initialization arguments: `auto_match`, `huge_tree`, If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code: ```python -from scrapling.default import Fetcher, StealthyFetcher, PlayWrightFetcher +from scrapling.defaults import Fetcher, StealthyFetcher, PlayWrightFetcher ``` then use it right away without initializing like: ```python From 7608d86bf2f4fd7b577d83fe91dd73bc384921c6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 11:24:47 +0200 Subject: [PATCH 04/13] Pumping up camoufox version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c360b7a..1ac7aaf 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( 'httpx[brotli,zstd]', 'playwright==1.48', # Temporary because currently All libraries that provide CDP patches doesn't support playwright 1.49 yet 'rebrowser-playwright', - 'camoufox>=0.3.10', + 'camoufox>=0.4.4', 'browserforge', ], python_requires=">=3.8", From 4dddebbe241f438bbe9683995947cddb91d9ffa9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 13:47:46 +0200 Subject: [PATCH 05/13] Update .bandit.yml --- .bandit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bandit.yml b/.bandit.yml index 57a5cc4..9d7a1f7 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -3,3 +3,5 @@ skips: - B311 - B320 - B410 +- B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only +- B403 # We are using pickle for tests only From 26bfa88e50db3f0e3f57a9b99f5914614328d94c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:01:30 +0200 Subject: [PATCH 06/13] Turn off threaded testing for Python 3.8 on tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 3640ae7..364e582 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ usedevelop = True changedir = tests deps = -r{toxinidir}/tests/requirements.txt +setenv = + py38: PYTEST_ADDOPTS = -p no:xdist commands = playwright install chromium playwright install-deps chromium firefox From 1a17b2ce46aacaee3176dd10403c8f04f81d6370 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:02:06 +0200 Subject: [PATCH 07/13] Sorting all library imports with `isort` --- benchmarks.py | 17 +++++++------- docs/Examples/selectorless_stackoverflow.py | 2 +- scrapling/__init__.py | 5 ++-- scrapling/core/_types.py | 5 ++-- scrapling/core/custom_types.py | 10 ++++---- scrapling/core/storage_adaptors.py | 12 +++++----- scrapling/core/translator.py | 11 ++++----- scrapling/core/utils.py | 13 +++++++---- scrapling/defaults.py | 2 +- scrapling/engines/__init__.py | 4 ++-- scrapling/engines/camo.py | 20 +++++++--------- scrapling/engines/pw.py | 23 +++++++----------- scrapling/engines/static.py | 7 +++--- scrapling/engines/toolbelt/__init__.py | 26 +++++---------------- scrapling/engines/toolbelt/custom.py | 5 ++-- scrapling/engines/toolbelt/fingerprints.py | 10 ++++---- scrapling/engines/toolbelt/navigation.py | 12 +++++----- scrapling/fetchers.py | 9 +++---- scrapling/parser.py | 23 +++++++++++------- setup.py | 2 +- tests/fetchers/test_camoufox.py | 1 + tests/fetchers/test_httpx.py | 1 + tests/fetchers/test_playwright.py | 1 + tests/parser/test_general.py | 4 +++- 24 files changed, 110 insertions(+), 115 deletions(-) diff --git a/benchmarks.py b/benchmarks.py index 8e46b37..de647d6 100644 --- a/benchmarks.py +++ b/benchmarks.py @@ -1,17 +1,18 @@ +import functools import time import timeit -import functools -import requests from statistics import mean -from scrapling import Adaptor -from parsel import Selector -from lxml import etree, html -from bs4 import BeautifulSoup -from pyquery import PyQuery as pq +import requests from autoscraper import AutoScraper -from selectolax.parser import HTMLParser +from bs4 import BeautifulSoup +from lxml import etree, html from mechanicalsoup import StatefulBrowser +from parsel import Selector +from pyquery import PyQuery as pq +from selectolax.parser import HTMLParser + +from scrapling import Adaptor large_html = '' + '
' * 5000 + '
' * 5000 + '' diff --git a/docs/Examples/selectorless_stackoverflow.py b/docs/Examples/selectorless_stackoverflow.py index f54007c..8619653 100644 --- a/docs/Examples/selectorless_stackoverflow.py +++ b/docs/Examples/selectorless_stackoverflow.py @@ -4,6 +4,7 @@ I only made this example to show how Scrapling features can be used to scrape a """ import requests + from scrapling import Adaptor response = requests.get('https://stackoverflow.com/questions/tagged/web-scraping?sort=MostVotes&filters=NoAcceptedAnswer&edited=true&pagesize=50&page=2') @@ -22,4 +23,3 @@ if first_question_title and first_question_author: # We will get all the rest of the titles/authors in the page depending on the first title and the first author we got above as a starting point for i, (title, author) in enumerate(zip(first_question_title.find_similar(), first_question_author.find_similar()), start=1): print(i, title.text, author.text) - diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 26d815d..4747bcf 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,7 +1,8 @@ # Declare top-level shortcuts -from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher +from scrapling.core.custom_types import AttributesHandler, TextHandler +from scrapling.fetchers import (CustomFetcher, Fetcher, PlayWrightFetcher, + StealthyFetcher) from scrapling.parser import Adaptor, Adaptors -from scrapling.core.custom_types import TextHandler, AttributesHandler __author__ = "Karim Shoair (karim.shoair@pm.me)" __version__ = "0.2.7" diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index f46dad4..8816b90 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -2,9 +2,8 @@ Type definitions for type checking purposes. """ -from typing import ( - Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING, Literal -) +from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, + List, Literal, Optional, Pattern, Tuple, Type, Union) try: from typing import Protocol diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 4670701..b8cb44f 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -1,13 +1,13 @@ import re -from types import MappingProxyType from collections.abc import Mapping +from types import MappingProxyType -from scrapling.core.utils import _is_iterable, flatten -from scrapling.core._types import Dict, List, Union, Pattern, SupportsIndex - -from orjson import loads, dumps +from orjson import dumps, loads from w3lib.html import replace_entities as _replace_entities +from scrapling.core._types import Dict, List, Pattern, SupportsIndex, Union +from scrapling.core.utils import _is_iterable, flatten + class TextHandler(str): """Extends standard Python string by adding more functionality""" diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.py index 675b46d..983e863 100644 --- a/scrapling/core/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -1,16 +1,16 @@ -import orjson -import sqlite3 import logging +import sqlite3 import threading -from hashlib import sha256 from abc import ABC, abstractmethod +from hashlib import sha256 + +import orjson +from lxml import html +from tldextract import extract as tld from scrapling.core._types import Dict, Optional, Union from scrapling.core.utils import _StorageTools, cache -from lxml import html -from tldextract import extract as tld - class StorageSystemMixin(ABC): # If you want to make your own storage system, you have to inherit from this diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index ec1b8ac..aa6211e 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -10,15 +10,14 @@ So you don't have to learn a new selectors/api method like what bs4 done with so import re -from w3lib.html import HTML5_WHITESPACE -from scrapling.core.utils import cache -from scrapling.core._types import Any, Optional, Protocol, Self - -from cssselect.xpath import ExpressionError -from cssselect.xpath import XPathExpr as OriginalXPathExpr from cssselect import HTMLTranslator as OriginalHTMLTranslator from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement +from cssselect.xpath import ExpressionError +from cssselect.xpath import XPathExpr as OriginalXPathExpr +from w3lib.html import HTML5_WHITESPACE +from scrapling.core._types import Any, Optional, Protocol, Self +from scrapling.core.utils import cache regex = f"[{HTML5_WHITESPACE}]+" replace_html5_whitespaces = re.compile(regex).sub diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index 3cf887c..35f8d0a 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -1,14 +1,17 @@ -import re import logging +import re from itertools import chain -# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code -from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache - -from scrapling.core._types import Dict, Iterable, Any, Union import orjson from lxml import html +from scrapling.core._types import Any, Dict, Iterable, Union + +# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code +# functools.cache is available on Python 3.9+ only so let's keep lru_cache +from functools import lru_cache as cache # isort:skip + + html_forbidden = {html.HtmlComment, } logging.basicConfig( level=logging.ERROR, diff --git a/scrapling/defaults.py b/scrapling/defaults.py index 79aa2ff..73618a4 100644 --- a/scrapling/defaults.py +++ b/scrapling/defaults.py @@ -1,4 +1,4 @@ -from .fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher +from .fetchers import Fetcher, PlayWrightFetcher, StealthyFetcher # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code Fetcher = Fetcher() diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index d91e20a..acdbeb0 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,7 +1,7 @@ from .camo import CamoufoxEngine -from .static import StaticEngine -from .pw import PlaywrightEngine from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS +from .pw import PlaywrightEngine +from .static import StaticEngine from .toolbelt import check_if_engine_usable __all__ = ['CamoufoxEngine', 'PlaywrightEngine'] diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 90d3378..2741206 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -1,20 +1,16 @@ import logging -from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal - -from scrapling.engines.toolbelt import ( - Response, - do_nothing, - StatusText, - get_os_name, - intercept_route, - check_type_validity, - construct_proxy_dict, - generate_convincing_referer, -) from camoufox import DefaultAddons from camoufox.sync_api import Camoufox +from scrapling.core._types import (Callable, Dict, List, Literal, Optional, + Union) +from scrapling.engines.toolbelt import (Response, StatusText, + check_type_validity, + construct_proxy_dict, do_nothing, + generate_convincing_referer, + get_os_name, intercept_route) + class CamoufoxEngine: def __init__( diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 818720b..7d15174 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -1,20 +1,15 @@ import json import logging -from scrapling.core._types import Union, Callable, Optional, List, Dict -from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY -from scrapling.engines.toolbelt import ( - Response, - do_nothing, - StatusText, - js_bypass_path, - intercept_route, - generate_headers, - construct_cdp_url, - check_type_validity, - construct_proxy_dict, - generate_convincing_referer, -) +from scrapling.core._types import Callable, Dict, List, Optional, Union +from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS, + NSTBROWSER_DEFAULT_QUERY) +from scrapling.engines.toolbelt import (Response, StatusText, + check_type_validity, construct_cdp_url, + construct_proxy_dict, do_nothing, + generate_convincing_referer, + generate_headers, intercept_route, + js_bypass_path) class PlaywrightEngine: diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index d6b5a6c..a091c4f 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,11 +1,12 @@ import logging -from scrapling.core._types import Union, Optional, Dict -from .toolbelt import Response, generate_convincing_referer, generate_headers - import httpx from httpx._models import Response as httpxResponse +from scrapling.core._types import Dict, Optional, Union + +from .toolbelt import Response, generate_convincing_referer, generate_headers + class StaticEngine: def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None): diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py index 15fd80c..595929c 100644 --- a/scrapling/engines/toolbelt/__init__.py +++ b/scrapling/engines/toolbelt/__init__.py @@ -1,20 +1,6 @@ -from .fingerprints import ( - get_os_name, - generate_headers, - generate_convincing_referer, -) -from .custom import ( - Response, - do_nothing, - StatusText, - BaseFetcher, - get_variable_name, - check_type_validity, - check_if_engine_usable, -) -from .navigation import ( - js_bypass_path, - intercept_route, - construct_cdp_url, - construct_proxy_dict, -) +from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable, + check_type_validity, do_nothing, get_variable_name) +from .fingerprints import (generate_convincing_referer, generate_headers, + get_os_name) +from .navigation import (construct_cdp_url, construct_proxy_dict, + intercept_route, js_bypass_path) diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 2c7cd94..6e321cc 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -5,10 +5,11 @@ import inspect import logging from email.message import Message +from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple, + Type, Union) from scrapling.core.custom_types import MappingProxyType +from scrapling.core.utils import cache, setup_basic_logging from scrapling.parser import Adaptor, SQLiteStorageSystem -from scrapling.core.utils import setup_basic_logging, cache -from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple class ResponseEncoding: diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 9cd337d..5600003 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -4,12 +4,12 @@ Functions related to generating headers and fingerprints generally import platform -from scrapling.core.utils import cache -from scrapling.core._types import Union, Dict - +from browserforge.fingerprints import Fingerprint, FingerprintGenerator +from browserforge.headers import Browser, HeaderGenerator from tldextract import extract -from browserforge.headers import HeaderGenerator, Browser -from browserforge.fingerprints import FingerprintGenerator, Fingerprint + +from scrapling.core._types import Dict, Union +from scrapling.core.utils import cache @cache(None, typed=True) diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index faf4402..2d24cac 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -2,16 +2,16 @@ Functions related to files and URLs """ -import os import logging -from urllib.parse import urlparse, urlencode - -from scrapling.core.utils import cache -from scrapling.core._types import Union, Dict, Optional -from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES +import os +from urllib.parse import urlencode, urlparse from playwright.sync_api import Route +from scrapling.core._types import Dict, Optional, Union +from scrapling.core.utils import cache +from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES + def intercept_route(route: Route) -> Union[Route, None]: """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES` diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 552f924..619f2f8 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -1,7 +1,8 @@ -from scrapling.core._types import Dict, Optional, Union, Callable, List, Literal - -from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing -from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable +from scrapling.core._types import (Callable, Dict, List, Literal, Optional, + Union) +from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine, + check_if_engine_usable) +from scrapling.engines.toolbelt import BaseFetcher, Response, do_nothing class Fetcher(BaseFetcher): diff --git a/scrapling/parser.py b/scrapling/parser.py index 79cfa14..daaa8c4 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,16 +1,23 @@ +import inspect import os import re -import inspect from difflib import SequenceMatcher -from scrapling.core.translator import HTMLTranslator -from scrapling.core.mixins import SelectorsGeneration -from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler -from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools -from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden, is_jsonable -from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable +from cssselect import SelectorError, SelectorSyntaxError +from cssselect import parse as split_selectors from lxml import etree, html -from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors + +from scrapling.core._types import (Any, Callable, Dict, Generator, Iterable, + List, Optional, Pattern, SupportsIndex, + Tuple, Union) +from scrapling.core.custom_types import (AttributesHandler, TextHandler, + TextHandlers) +from scrapling.core.mixins import SelectorsGeneration +from scrapling.core.storage_adaptors import (SQLiteStorageSystem, + StorageSystemMixin, _StorageTools) +from scrapling.core.translator import HTMLTranslator +from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, + is_jsonable, logging, setup_basic_logging) class Adaptor(SelectorsGeneration): diff --git a/setup.py b/setup.py index 1ac7aaf..6952887 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() diff --git a/tests/fetchers/test_camoufox.py b/tests/fetchers/test_camoufox.py index 5a0fe17..fcbf3b7 100644 --- a/tests/fetchers/test_camoufox.py +++ b/tests/fetchers/test_camoufox.py @@ -1,4 +1,5 @@ import unittest + import pytest_httpbin from scrapling import StealthyFetcher diff --git a/tests/fetchers/test_httpx.py b/tests/fetchers/test_httpx.py index 2fcd585..1a5cc02 100644 --- a/tests/fetchers/test_httpx.py +++ b/tests/fetchers/test_httpx.py @@ -1,4 +1,5 @@ import unittest + import pytest_httpbin from scrapling import Fetcher diff --git a/tests/fetchers/test_playwright.py b/tests/fetchers/test_playwright.py index 8f67b3d..dda30e0 100644 --- a/tests/fetchers/test_playwright.py +++ b/tests/fetchers/test_playwright.py @@ -1,4 +1,5 @@ import unittest + import pytest_httpbin from scrapling import PlayWrightFetcher diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 1f0dfe2..ea1fb78 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -1,9 +1,11 @@ import pickle import unittest -from scrapling import Adaptor + from cssselect import SelectorError, SelectorSyntaxError +from scrapling import Adaptor + class TestParser(unittest.TestCase): def setUp(self): From f81e3b3cc3177264170b293c8890722a25096098 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:08:18 +0200 Subject: [PATCH 08/13] Turn off threaded testing for Python 3.8 on tox part 2 :) --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 364e582..7da4eaa 100644 --- a/tox.ini +++ b/tox.ini @@ -11,13 +11,12 @@ usedevelop = True changedir = tests deps = -r{toxinidir}/tests/requirements.txt -setenv = - py38: PYTEST_ADDOPTS = -p no:xdist commands = playwright install chromium playwright install-deps chromium firefox camoufox fetch --browserforge - pytest --cov=scrapling --cov-report=xml -n auto + py38: pytest --cov=scrapling --cov-report=xml + py{39,310,311,312,313}: pytest --cov=scrapling --cov-report=xml -n auto [testenv:pre-commit] basepython = python3 From b38c38b03b3dca770cb68f3222e2ac9a843bd08c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:14:59 +0200 Subject: [PATCH 09/13] Forcing Tox to always use pytest config file --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 7da4eaa..cc08e05 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,8 @@ commands = playwright install chromium playwright install-deps chromium firefox camoufox fetch --browserforge - py38: pytest --cov=scrapling --cov-report=xml - py{39,310,311,312,313}: pytest --cov=scrapling --cov-report=xml -n auto + py38: pytest --config-file=../pytest.ini --cov=scrapling --cov-report=xml + py{39,310,311,312,313}: pytest --config-file=../pytest.ini --cov=scrapling --cov-report=xml -n auto [testenv:pre-commit] basepython = python3 From 8321cfc1807dafd41676dabd0f2c07d70730b149 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:28:10 +0200 Subject: [PATCH 10/13] Add vermin hook to check for minimum python version --- .pre-commit-config.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b22d4a..9e3cf04 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,19 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.8 + rev: 1.8.0 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - - id: isort \ No newline at end of file + - id: isort +- repo: https://github.com/netromdk/vermin + rev: v1.6.0 + hooks: + - id: vermin + args: ['-t=3.8-', '--violations', '--eval-annotations', '--no-tips'] From f7916e504e953d086117151ad1142f0a14c3709d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 14:32:42 +0200 Subject: [PATCH 11/13] Update tox.ini --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cc08e05..28b09e1 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,8 @@ commands = playwright install chromium playwright install-deps chromium firefox camoufox fetch --browserforge - py38: pytest --config-file=../pytest.ini --cov=scrapling --cov-report=xml - py{39,310,311,312,313}: pytest --config-file=../pytest.ini --cov=scrapling --cov-report=xml -n auto + py38: pytest --config-file=pytest.ini --cov=scrapling --cov-report=xml + py{39,310,311,312,313}: pytest --config-file=pytest.ini --cov=scrapling --cov-report=xml -n auto [testenv:pre-commit] basepython = python3 From a3a4f1ff19d9d24067ba852671282627318acb93 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 18:01:35 +0200 Subject: [PATCH 12/13] A cleanup script to use with development --- cleanup.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 cleanup.py diff --git a/cleanup.py b/cleanup.py new file mode 100644 index 0000000..8a1ed3a --- /dev/null +++ b/cleanup.py @@ -0,0 +1,42 @@ +import shutil +from pathlib import Path + + +# Clean up after installing for local development +def clean(): + # Get the current directory + base_dir = Path.cwd() + + # Directories and patterns to clean + cleanup_patterns = [ + 'build', + 'dist', + '*.egg-info', + '__pycache__', + '.eggs', + '.pytest_cache' + ] + + # Clean directories + for pattern in cleanup_patterns: + for path in base_dir.glob(pattern): + try: + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + print(f"Removed: {path}") + except Exception as e: + print(f"Could not remove {path}: {e}") + + # Remove compiled Python files + for path in base_dir.rglob('*.py[co]'): + try: + path.unlink() + print(f"Removed compiled file: {path}") + except Exception as e: + print(f"Could not remove {path}: {e}") + + +if __name__ == '__main__': + clean() From c481a1c8ad633edca32db3631d4d800209ca493e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 30 Nov 2024 18:13:19 +0200 Subject: [PATCH 13/13] Pumping version up to 0.2.8 --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 4747bcf..9240821 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -5,7 +5,7 @@ from scrapling.fetchers import (CustomFetcher, Fetcher, PlayWrightFetcher, from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.7" +__version__ = "0.2.8" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index 1aa408a..84169d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.7 +version = 0.2.8 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python. diff --git a/setup.py b/setup.py index 6952887..0a29929 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.7", + version="0.2.8", description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It simplifies the process of extracting data from websites, even when they undergo structural changes, and offers impressive speed improvements over many popular scraping tools.""",