freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #21 from D4Vinci/dev

Browse Source 
v0.2.8
This commit is contained in:
Karim shoair
2024-11-30 18:16:06 +02:00
committed by GitHub
parent d6cc07deaa c481a1c8ad
commit 012820c635
32 changed files with 188 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bandit.yml
+2
View File
@@ -3,3 +3,5 @@ skips:
- B311 - B311
- B320 - B320
- B410 - B410
- B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only
- B403 # We are using pickle for tests only
.flake8
+2 -2
View File
@@ -1,3 +1,3 @@
[flake8] [flake8]
ignore = E501 # line too long ignore = E501, F401
exclude = .git,__pycache__,docs,.github,build,dist exclude = .git,.venv,__pycache__,docs,.github,build,dist,tests,benchmarks.py
.github/workflows/tests.yml
+5 -1
View File
@@ -1,5 +1,9 @@
name: Tests name: Tests
on: [push] on:
push:
branches:
- main
- dev
concurrency: concurrency:
group: ${{github.workflow}}-${{ github.ref }} group: ${{github.workflow}}-${{ github.ref }}
.pre-commit-config.yaml
+8 -3
View File
@@ -1,14 +1,19 @@
repos: repos:
- repo: https://github.com/PyCQA/bandit - repo: https://github.com/PyCQA/bandit
rev: 1.7.8 rev: 1.8.0
hooks: hooks:
- id: bandit - id: bandit
args: [-r, -c, .bandit.yml] args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8 - repo: https://github.com/PyCQA/flake8
rev: 7.0.0 rev: 7.1.1
hooks: hooks:
- id: flake8 - id: flake8
- repo: https://github.com/pycqa/isort - repo: https://github.com/pycqa/isort
rev: 5.13.2 rev: 5.13.2
hooks: hooks:
- id: isort - id: isort
- repo: https://github.com/netromdk/vermin
rev: v1.6.0
hooks:
- id: vermin
args: ['-t=3.8-', '--violations', '--eval-annotations', '--no-tips']
README.md
+2 -2
View File
@@ -6,7 +6,7 @@ Dealing with failing web scrapers due to anti-bot protections or website changes
Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. For both beginners and experts, Scrapling provides powerful features while maintaining simplicity. Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. For both beginners and experts, Scrapling provides powerful features while maintaining simplicity.
```python ```python
>> from scrapling.default import Fetcher, StealthyFetcher, PlayWrightFetcher >> from scrapling.defaults import Fetcher, StealthyFetcher, PlayWrightFetcher
# Fetch websites' source under the radar! # Fetch websites' source under the radar!
>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) >> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)
>> print(page.status) >> print(page.status)
@@ -223,7 +223,7 @@ All of them can take these initialization arguments: `auto_match`, `huge_tree`,
If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code: If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code:
```python ```python
from scrapling.default import Fetcher, StealthyFetcher, PlayWrightFetcher from scrapling.defaults import Fetcher, StealthyFetcher, PlayWrightFetcher
``` ```
then use it right away without initializing like: then use it right away without initializing like:
```python ```python
benchmarks.py
+9 -8
View File
@@ -1,17 +1,18 @@
import functools
import time import time
import timeit import timeit
import functools
import requests
from statistics import mean from statistics import mean
from scrapling import Adaptor import requests
from parsel import Selector
from lxml import etree, html
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from autoscraper import AutoScraper from autoscraper import AutoScraper
from selectolax.parser import HTMLParser from bs4 import BeautifulSoup
from lxml import etree, html
from mechanicalsoup import StatefulBrowser from mechanicalsoup import StatefulBrowser
from parsel import Selector
from pyquery import PyQuery as pq
from selectolax.parser import HTMLParser
from scrapling import Adaptor
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>' large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
cleanup.py
+42
View File
@@ -0,0 +1,42 @@
import shutil
from pathlib import Path
# Clean up after installing for local development
def clean():
# Get the current directory
base_dir = Path.cwd()
# Directories and patterns to clean
cleanup_patterns = [
'build',
'dist',
'*.egg-info',
'__pycache__',
'.eggs',
'.pytest_cache'
]
# Clean directories
for pattern in cleanup_patterns:
for path in base_dir.glob(pattern):
try:
if path.is_dir():
shutil.rmtree(path)
else:
path.unlink()
print(f"Removed: {path}")
except Exception as e:
print(f"Could not remove {path}: {e}")
# Remove compiled Python files
for path in base_dir.rglob('*.py[co]'):
try:
path.unlink()
print(f"Removed compiled file: {path}")
except Exception as e:
print(f"Could not remove {path}: {e}")
if __name__ == '__main__':
clean()
docs/Examples/selectorless_stackoverflow.py
+1 -1
View File
@@ -4,6 +4,7 @@ I only made this example to show how Scrapling features can be used to scrape a
""" """
import requests import requests
from scrapling import Adaptor from scrapling import Adaptor
response = requests.get('https://stackoverflow.com/questions/tagged/web-scraping?sort=MostVotes&filters=NoAcceptedAnswer&edited=true&pagesize=50&page=2') response = requests.get('https://stackoverflow.com/questions/tagged/web-scraping?sort=MostVotes&filters=NoAcceptedAnswer&edited=true&pagesize=50&page=2')
@@ -22,4 +23,3 @@ if first_question_title and first_question_author:
# We will get all the rest of the titles/authors in the page depending on the first title and the first author we got above as a starting point # We will get all the rest of the titles/authors in the page depending on the first title and the first author we got above as a starting point
for i, (title, author) in enumerate(zip(first_question_title.find_similar(), first_question_author.find_similar()), start=1): for i, (title, author) in enumerate(zip(first_question_title.find_similar(), first_question_author.find_similar()), start=1):
print(i, title.text, author.text) print(i, title.text, author.text)
scrapling/__init__.py
+4 -3
View File
@@ -1,10 +1,11 @@
# Declare top-level shortcuts # Declare top-level shortcuts
from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher from scrapling.core.custom_types import AttributesHandler, TextHandler
from scrapling.fetchers import (CustomFetcher, Fetcher, PlayWrightFetcher,
StealthyFetcher)
from scrapling.parser import Adaptor, Adaptors from scrapling.parser import Adaptor, Adaptors
from scrapling.core.custom_types import TextHandler, AttributesHandler
__author__ = "Karim Shoair (karim.shoair@pm.me)" __author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.2.7" __version__ = "0.2.8"
__copyright__ = "Copyright (c) 2024 Karim Shoair" __copyright__ = "Copyright (c) 2024 Karim Shoair"
scrapling/core/_types.py
+2 -3
View File
@@ -2,9 +2,8 @@
Type definitions for type checking purposes. Type definitions for type checking purposes.
""" """
from typing import ( from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING, Literal List, Literal, Optional, Pattern, Tuple, Type, Union)
)
try: try:
from typing import Protocol from typing import Protocol
scrapling/core/custom_types.py
+5 -5
View File
@@ -1,13 +1,13 @@
import re import re
from types import MappingProxyType
from collections.abc import Mapping from collections.abc import Mapping
from types import MappingProxyType
from scrapling.core.utils import _is_iterable, flatten from orjson import dumps, loads
from scrapling.core._types import Dict, List, Union, Pattern, SupportsIndex
from orjson import loads, dumps
from w3lib.html import replace_entities as _replace_entities from w3lib.html import replace_entities as _replace_entities
from scrapling.core._types import Dict, List, Pattern, SupportsIndex, Union
from scrapling.core.utils import _is_iterable, flatten
class TextHandler(str): class TextHandler(str):
"""Extends standard Python string by adding more functionality""" """Extends standard Python string by adding more functionality"""
scrapling/core/storage_adaptors.py
+6 -6
View File
@@ -1,16 +1,16 @@
import orjson
import sqlite3
import logging import logging
import sqlite3
import threading import threading
from hashlib import sha256
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from hashlib import sha256
import orjson
from lxml import html
from tldextract import extract as tld
from scrapling.core._types import Dict, Optional, Union from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import _StorageTools, cache from scrapling.core.utils import _StorageTools, cache
from lxml import html
from tldextract import extract as tld
class StorageSystemMixin(ABC): class StorageSystemMixin(ABC):
# If you want to make your own storage system, you have to inherit from this # If you want to make your own storage system, you have to inherit from this
scrapling/core/translator.py
+5 -6
View File
@@ -10,15 +10,14 @@ So you don't have to learn a new selectors/api method like what bs4 done with so
import re import re
from w3lib.html import HTML5_WHITESPACE
from scrapling.core.utils import cache
from scrapling.core._types import Any, Optional, Protocol, Self
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from cssselect import HTMLTranslator as OriginalHTMLTranslator from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from w3lib.html import HTML5_WHITESPACE
from scrapling.core._types import Any, Optional, Protocol, Self
from scrapling.core.utils import cache
regex = f"[{HTML5_WHITESPACE}]+" regex = f"[{HTML5_WHITESPACE}]+"
replace_html5_whitespaces = re.compile(regex).sub replace_html5_whitespaces = re.compile(regex).sub
scrapling/core/utils.py
+15 -12
View File
@@ -1,22 +1,25 @@
import re
import logging import logging
import re
from itertools import chain from itertools import chain
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache
from scrapling.core._types import Dict, Iterable, Any, Union
import orjson import orjson
from lxml import html from lxml import html
from scrapling.core._types import Any, Dict, Iterable, Union
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
# functools.cache is available on Python 3.9+ only so let's keep lru_cache
from functools import lru_cache as cache # isort:skip
html_forbidden = {html.HtmlComment, } html_forbidden = {html.HtmlComment, }
logging.basicConfig( logging.basicConfig(
level=logging.ERROR, level=logging.ERROR,
format='%(asctime)s - %(levelname)s - %(message)s', format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[ handlers=[
logging.StreamHandler() logging.StreamHandler()
] ]
) )
def is_jsonable(content: Union[bytes, str]) -> bool: def is_jsonable(content: Union[bytes, str]) -> bool:
@@ -94,7 +97,7 @@ class _StorageTools:
parent = element.getparent() parent = element.getparent()
return tuple( return tuple(
(element.tag,) if parent is None else ( (element.tag,) if parent is None else (
cls._get_element_path(parent) + (element.tag,) cls._get_element_path(parent) + (element.tag,)
) )
) )
scrapling/defaults.py
+1 -1
View File
@@ -1,4 +1,4 @@
from .fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher from .fetchers import Fetcher, PlayWrightFetcher, StealthyFetcher
# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
Fetcher = Fetcher() Fetcher = Fetcher()
scrapling/engines/__init__.py
+2 -2
View File
@@ -1,7 +1,7 @@
from .camo import CamoufoxEngine from .camo import CamoufoxEngine
from .static import StaticEngine
from .pw import PlaywrightEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .pw import PlaywrightEngine
from .static import StaticEngine
from .toolbelt import check_if_engine_usable from .toolbelt import check_if_engine_usable
__all__ = ['CamoufoxEngine', 'PlaywrightEngine'] __all__ = ['CamoufoxEngine', 'PlaywrightEngine']
scrapling/engines/camo.py
+8 -12
View File
@@ -1,20 +1,16 @@
import logging import logging
from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
from scrapling.engines.toolbelt import (
Response,
do_nothing,
StatusText,
get_os_name,
intercept_route,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
)
from camoufox import DefaultAddons from camoufox import DefaultAddons
from camoufox.sync_api import Camoufox from camoufox.sync_api import Camoufox
from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
Union)
from scrapling.engines.toolbelt import (Response, StatusText,
check_type_validity,
construct_proxy_dict, do_nothing,
generate_convincing_referer,
get_os_name, intercept_route)
class CamoufoxEngine: class CamoufoxEngine:
def __init__( def __init__(
scrapling/engines/pw.py
+9 -14
View File
@@ -1,20 +1,15 @@
import json import json
import logging import logging
from scrapling.core._types import Union, Callable, Optional, List, Dict
from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY from scrapling.core._types import Callable, Dict, List, Optional, Union
from scrapling.engines.toolbelt import ( from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
Response, NSTBROWSER_DEFAULT_QUERY)
do_nothing, from scrapling.engines.toolbelt import (Response, StatusText,
StatusText, check_type_validity, construct_cdp_url,
js_bypass_path, construct_proxy_dict, do_nothing,
intercept_route, generate_convincing_referer,
generate_headers, generate_headers, intercept_route,
construct_cdp_url, js_bypass_path)
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
)
class PlaywrightEngine: class PlaywrightEngine:
scrapling/engines/static.py
+4 -3
View File
@@ -1,11 +1,12 @@
import logging import logging
from scrapling.core._types import Union, Optional, Dict
from .toolbelt import Response, generate_convincing_referer, generate_headers
import httpx import httpx
from httpx._models import Response as httpxResponse from httpx._models import Response as httpxResponse
from scrapling.core._types import Dict, Optional, Union
from .toolbelt import Response, generate_convincing_referer, generate_headers
class StaticEngine: class StaticEngine:
def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None): def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
scrapling/engines/toolbelt/__init__.py
+6 -20
View File
@@ -1,20 +1,6 @@
from .fingerprints import ( from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,
get_os_name, check_type_validity, do_nothing, get_variable_name)
generate_headers, from .fingerprints import (generate_convincing_referer, generate_headers,
generate_convincing_referer, get_os_name)
) from .navigation import (construct_cdp_url, construct_proxy_dict,
from .custom import ( intercept_route, js_bypass_path)
Response,
do_nothing,
StatusText,
BaseFetcher,
get_variable_name,
check_type_validity,
check_if_engine_usable,
)
from .navigation import (
js_bypass_path,
intercept_route,
construct_cdp_url,
construct_proxy_dict,
)
scrapling/engines/toolbelt/custom.py
+3 -2
View File
@@ -5,10 +5,11 @@ import inspect
import logging import logging
from email.message import Message from email.message import Message
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
Type, Union)
from scrapling.core.custom_types import MappingProxyType from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import cache, setup_basic_logging
from scrapling.parser import Adaptor, SQLiteStorageSystem from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.core.utils import setup_basic_logging, cache
from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple
class ResponseEncoding: class ResponseEncoding:
scrapling/engines/toolbelt/fingerprints.py
+5 -5
View File
@@ -4,12 +4,12 @@ Functions related to generating headers and fingerprints generally
import platform import platform
from scrapling.core.utils import cache from browserforge.fingerprints import Fingerprint, FingerprintGenerator
from scrapling.core._types import Union, Dict from browserforge.headers import Browser, HeaderGenerator
from tldextract import extract from tldextract import extract
from browserforge.headers import HeaderGenerator, Browser
from browserforge.fingerprints import FingerprintGenerator, Fingerprint from scrapling.core._types import Dict, Union
from scrapling.core.utils import cache
@cache(None, typed=True) @cache(None, typed=True)
scrapling/engines/toolbelt/navigation.py
+7 -7
View File
@@ -2,16 +2,16 @@
Functions related to files and URLs Functions related to files and URLs
""" """
import os
import logging import logging
from urllib.parse import urlparse, urlencode import os
from urllib.parse import urlencode, urlparse
from scrapling.core.utils import cache
from scrapling.core._types import Union, Dict, Optional
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
from playwright.sync_api import Route from playwright.sync_api import Route
from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import cache
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
def intercept_route(route: Route) -> Union[Route, None]: def intercept_route(route: Route) -> Union[Route, None]:
"""This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES` """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
@@ -43,7 +43,7 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict
} }
except ValueError: except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc... # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError(f'The proxy argument\'s string is in invalid format!') raise TypeError('The proxy argument\'s string is in invalid format!')
elif isinstance(proxy_string, dict): elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', ) valid_keys = ('server', 'username', 'password', )
scrapling/fetchers.py
+5 -4
View File
@@ -1,7 +1,8 @@
from scrapling.core._types import Dict, Optional, Union, Callable, List, Literal from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
Union)
from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine,
from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable check_if_engine_usable)
from scrapling.engines.toolbelt import BaseFetcher, Response, do_nothing
class Fetcher(BaseFetcher): class Fetcher(BaseFetcher):
scrapling/parser.py
+15 -8
View File
@@ -1,16 +1,23 @@
import inspect
import os import os
import re import re
import inspect
from difflib import SequenceMatcher from difflib import SequenceMatcher
from scrapling.core.translator import HTMLTranslator from cssselect import SelectorError, SelectorSyntaxError
from scrapling.core.mixins import SelectorsGeneration from cssselect import parse as split_selectors
from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden, is_jsonable
from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable
from lxml import etree, html from lxml import etree, html
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
from scrapling.core._types import (Any, Callable, Dict, Generator, Iterable,
List, Optional, Pattern, SupportsIndex,
Tuple, Union)
from scrapling.core.custom_types import (AttributesHandler, TextHandler,
TextHandlers)
from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
StorageSystemMixin, _StorageTools)
from scrapling.core.translator import HTMLTranslator
from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
is_jsonable, logging, setup_basic_logging)
class Adaptor(SelectorsGeneration): class Adaptor(SelectorsGeneration):
setup.cfg
+1 -1
View File
@@ -1,6 +1,6 @@
[metadata] [metadata]
name = scrapling name = scrapling
version = 0.2.7 version = 0.2.8
author = Karim Shoair author = Karim Shoair
author_email = karim.shoair@pm.me author_email = karim.shoair@pm.me
description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python. description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python.
setup.py
+6 -6
View File
@@ -1,4 +1,4 @@
from setuptools import setup, find_packages from setuptools import find_packages, setup
with open("README.md", "r", encoding="utf-8") as fh: with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read() long_description = fh.read()
@@ -6,10 +6,10 @@ with open("README.md", "r", encoding="utf-8") as fh:
setup( setup(
name="scrapling", name="scrapling",
version="0.2.7", version="0.2.8",
description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It
simplifies the process of extracting data from websites, even when they undergo structural changes, and offers simplifies the process of extracting data from websites, even when they undergo structural changes, and offers
impressive speed improvements over many popular scraping tools.""", impressive speed improvements over many popular scraping tools.""",
long_description=long_description, long_description=long_description,
long_description_content_type="text/markdown", long_description_content_type="text/markdown",
author="Karim Shoair", author="Karim Shoair",
@@ -57,7 +57,7 @@ setup(
'httpx[brotli,zstd]', 'httpx[brotli,zstd]',
'playwright==1.48', # Temporary because currently All libraries that provide CDP patches doesn't support playwright 1.49 yet 'playwright==1.48', # Temporary because currently All libraries that provide CDP patches doesn't support playwright 1.49 yet
'rebrowser-playwright', 'rebrowser-playwright',
'camoufox>=0.3.10', 'camoufox>=0.4.4',
'browserforge', 'browserforge',
], ],
python_requires=">=3.8", python_requires=">=3.8",
tests/fetchers/test_camoufox.py
+1
View File
@@ -1,4 +1,5 @@
import unittest import unittest
import pytest_httpbin import pytest_httpbin
from scrapling import StealthyFetcher from scrapling import StealthyFetcher
tests/fetchers/test_httpx.py
+1
View File
@@ -1,4 +1,5 @@
import unittest import unittest
import pytest_httpbin import pytest_httpbin
from scrapling import Fetcher from scrapling import Fetcher
tests/fetchers/test_playwright.py
+1
View File
@@ -1,4 +1,5 @@
import unittest import unittest
import pytest_httpbin import pytest_httpbin
from scrapling import PlayWrightFetcher from scrapling import PlayWrightFetcher
tests/parser/test_general.py
+3 -1
View File
@@ -1,9 +1,11 @@
import pickle import pickle
import unittest import unittest
from scrapling import Adaptor
from cssselect import SelectorError, SelectorSyntaxError from cssselect import SelectorError, SelectorSyntaxError
from scrapling import Adaptor
class TestParser(unittest.TestCase): class TestParser(unittest.TestCase):
def setUp(self): def setUp(self):
tox.ini
+2 -1
View File
@@ -15,7 +15,8 @@ commands =
playwright install chromium playwright install chromium
playwright install-deps chromium firefox playwright install-deps chromium firefox
camoufox fetch --browserforge camoufox fetch --browserforge
pytest --cov=scrapling --cov-report=xml -n auto py38: pytest --config-file=pytest.ini --cov=scrapling --cov-report=xml
py{39,310,311,312,313}: pytest --config-file=pytest.ini --cov=scrapling --cov-report=xml -n auto
[testenv:pre-commit] [testenv:pre-commit]
basepython = python3 basepython = python3