freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
-3
View File
@@ -1,3 +0,0 @@
[flake8]
ignore = E501, F401
exclude = .git,.venv,__pycache__,docs,.github,build,dist,tests,benchmarks.py
.pre-commit-config.yaml
+9 -8
View File
@@ -1,17 +1,18 @@
repos: repos:
- repo: https://github.com/PyCQA/bandit - repo: https://github.com/PyCQA/bandit
rev: 1.8.0 rev: 1.8.3
hooks: hooks:
- id: bandit - id: bandit
args: [-r, -c, .bandit.yml] args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8 - repo: https://github.com/astral-sh/ruff-pre-commit
rev: 7.1.1 # Ruff version.
rev: v0.11.5
hooks: hooks:
- id: flake8 # Run the linter.
- repo: https://github.com/pycqa/isort - id: ruff
rev: 5.13.2 args: [ --fix ]
hooks: # Run the formatter.
- id: isort - id: ruff-format
- repo: https://github.com/netromdk/vermin - repo: https://github.com/netromdk/vermin
rev: v1.6.0 rev: v1.6.0
hooks: hooks:
benchmarks.py
+36 -24
View File
@@ -14,19 +14,27 @@ from selectolax.parser import HTMLParser
from scrapling import Adaptor from scrapling import Adaptor
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>' large_html = (
"<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>"
)
def benchmark(func): def benchmark(func):
@functools.wraps(func) @functools.wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
benchmark_name = func.__name__.replace('test_', '').replace('_', ' ') benchmark_name = func.__name__.replace("test_", "").replace("_", " ")
print(f"-> {benchmark_name}", end=" ", flush=True) print(f"-> {benchmark_name}", end=" ", flush=True)
# Warm-up phase # Warm-up phase
timeit.repeat(lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals()) timeit.repeat(
lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals()
)
# Measure time (1 run, repeat 100 times, take average) # Measure time (1 run, repeat 100 times, take average)
times = timeit.repeat( times = timeit.repeat(
lambda: func(*args, **kwargs), number=1, repeat=100, globals=globals(), timer=time.process_time lambda: func(*args, **kwargs),
number=1,
repeat=100,
globals=globals(),
timer=time.process_time,
) )
min_time = round(mean(times) * 1000, 2) # Convert to milliseconds min_time = round(mean(times) * 1000, 2) # Convert to milliseconds
print(f"average execution time: {min_time} ms") print(f"average execution time: {min_time} ms")
@@ -42,23 +50,24 @@ def test_lxml():
for e in etree.fromstring( for e in etree.fromstring(
large_html, large_html,
# Scrapling and Parsel use the same parser inside so this is just to make it fair # Scrapling and Parsel use the same parser inside so this is just to make it fair
parser=html.HTMLParser(recover=True, huge_tree=True) parser=html.HTMLParser(recover=True, huge_tree=True),
).cssselect('.item')] ).cssselect(".item")
]
@benchmark @benchmark
def test_bs4_lxml(): def test_bs4_lxml():
return [e.text for e in BeautifulSoup(large_html, 'lxml').select('.item')] return [e.text for e in BeautifulSoup(large_html, "lxml").select(".item")]
@benchmark @benchmark
def test_bs4_html5lib(): def test_bs4_html5lib():
return [e.text for e in BeautifulSoup(large_html, 'html5lib').select('.item')] return [e.text for e in BeautifulSoup(large_html, "html5lib").select(".item")]
@benchmark @benchmark
def test_pyquery(): def test_pyquery():
return [e.text() for e in pq(large_html)('.item').items()] return [e.text() for e in pq(large_html)(".item").items()]
@benchmark @benchmark
@@ -66,33 +75,33 @@ def test_scrapling():
# No need to do `.extract()` like parsel to extract text # No need to do `.extract()` like parsel to extract text
# Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]` # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]`
# for obvious reasons, of course. # for obvious reasons, of course.
return Adaptor(large_html, auto_match=False).css('.item::text') return Adaptor(large_html, auto_match=False).css(".item::text")
@benchmark @benchmark
def test_parsel(): def test_parsel():
return Selector(text=large_html).css('.item::text').extract() return Selector(text=large_html).css(".item::text").extract()
@benchmark @benchmark
def test_mechanicalsoup(): def test_mechanicalsoup():
browser = StatefulBrowser() browser = StatefulBrowser()
browser.open_fake_page(large_html) browser.open_fake_page(large_html)
return [e.text for e in browser.page.select('.item')] return [e.text for e in browser.page.select(".item")]
@benchmark @benchmark
def test_selectolax(): def test_selectolax():
return [node.text() for node in HTMLParser(large_html).css('.item')] return [node.text() for node in HTMLParser(large_html).css(".item")]
def display(results): def display(results):
# Sort and display results # Sort and display results
sorted_results = sorted(results.items(), key=lambda x: x[1]) # Sort by time sorted_results = sorted(results.items(), key=lambda x: x[1]) # Sort by time
scrapling_time = results['Scrapling'] scrapling_time = results["Scrapling"]
print("\nRanked Results (fastest to slowest):") print("\nRanked Results (fastest to slowest):")
print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling") print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling")
print('-' * 50) print("-" * 50)
for i, (test_name, test_time) in enumerate(sorted_results, 1): for i, (test_name, test_time) in enumerate(sorted_results, 1):
compare = round(test_time / scrapling_time, 3) compare = round(test_time / scrapling_time, 3)
print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}") print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}")
@@ -102,25 +111,28 @@ def display(results):
def test_scrapling_text(request_html): def test_scrapling_text(request_html):
# Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster # Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
return [ return [
element.text for element in Adaptor( element.text
request_html, auto_match=False for element in Adaptor(request_html, auto_match=False)
).find_by_text('Tipping the Velvet', first_match=True).find_similar(ignore_attributes=['title']) .find_by_text("Tipping the Velvet", first_match=True)
.find_similar(ignore_attributes=["title"])
] ]
@benchmark @benchmark
def test_autoscraper(request_html): def test_autoscraper(request_html):
# autoscraper by default returns elements text # autoscraper by default returns elements text
return AutoScraper().build(html=request_html, wanted_list=['Tipping the Velvet']) return AutoScraper().build(html=request_html, wanted_list=["Tipping the Velvet"])
if __name__ == "__main__": if __name__ == "__main__":
print(' Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n') print(
" Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n"
)
results1 = { results1 = {
"Raw Lxml": test_lxml(), "Raw Lxml": test_lxml(),
"Parsel/Scrapy": test_parsel(), "Parsel/Scrapy": test_parsel(),
"Scrapling": test_scrapling(), "Scrapling": test_scrapling(),
'Selectolax': test_selectolax(), "Selectolax": test_selectolax(),
"PyQuery": test_pyquery(), "PyQuery": test_pyquery(),
"BS4 with Lxml": test_bs4_lxml(), "BS4 with Lxml": test_bs4_lxml(),
"MechanicalSoup": test_mechanicalsoup(), "MechanicalSoup": test_mechanicalsoup(),
@@ -128,10 +140,10 @@ if __name__ == "__main__":
} }
display(results1) display(results1)
print('\n' + "="*25) print("\n" + "=" * 25)
req = requests.get('https://books.toscrape.com/index.html') req = requests.get("https://books.toscrape.com/index.html")
print( print(
' Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n' " Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n"
) )
results2 = { results2 = {
"Scrapling": test_scrapling_text(req.text), "Scrapling": test_scrapling_text(req.text),
cleanup.py
+8 -8
View File
@@ -9,12 +9,12 @@ def clean():
# Directories and patterns to clean # Directories and patterns to clean
cleanup_patterns = [ cleanup_patterns = [
'build', "build",
'dist', "dist",
'*.egg-info', "*.egg-info",
'__pycache__', "__pycache__",
'.eggs', ".eggs",
'.pytest_cache' ".pytest_cache",
] ]
# Clean directories # Clean directories
@@ -30,7 +30,7 @@ def clean():
print(f"Could not remove {path}: {e}") print(f"Could not remove {path}: {e}")
# Remove compiled Python files # Remove compiled Python files
for path in base_dir.rglob('*.py[co]'): for path in base_dir.rglob("*.py[co]"):
try: try:
path.unlink() path.unlink()
print(f"Removed compiled file: {path}") print(f"Removed compiled file: {path}")
@@ -38,5 +38,5 @@ def clean():
print(f"Could not remove {path}: {e}") print(f"Could not remove {path}: {e}")
if __name__ == '__main__': if __name__ == "__main__":
clean() clean()
ruff.toml
+22
View File
@@ -0,0 +1,22 @@
exclude = [
".git",
".venv",
"__pycache__",
"docs",
".github",
"build",
"dist",
"tests",
"benchmarks.py",
]
# Assume Python 3.9
target-version = "py39"
[lint]
select = ["E", "F", "W"]
ignore = ["E501", "F401"]
[format]
# Like Black, use double quotes for strings.
quote-style = "double"
scrapling/__init__.py
+19 -11
View File
@@ -1,4 +1,3 @@
__author__ = "Karim Shoair (karim.shoair@pm.me)" __author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.2.99" __version__ = "0.2.99"
__copyright__ = "Copyright (c) 2024 Karim Shoair" __copyright__ = "Copyright (c) 2024 Karim Shoair"
@@ -7,35 +6,44 @@ __copyright__ = "Copyright (c) 2024 Karim Shoair"
# A lightweight approach to create lazy loader for each import for backward compatibility # A lightweight approach to create lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used) # This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name): def __getattr__(name):
if name == 'Fetcher': if name == "Fetcher":
from scrapling.fetchers import Fetcher as cls from scrapling.fetchers import Fetcher as cls
return cls return cls
elif name == 'Adaptor': elif name == "Adaptor":
from scrapling.parser import Adaptor as cls from scrapling.parser import Adaptor as cls
return cls return cls
elif name == 'Adaptors': elif name == "Adaptors":
from scrapling.parser import Adaptors as cls from scrapling.parser import Adaptors as cls
return cls return cls
elif name == 'AttributesHandler': elif name == "AttributesHandler":
from scrapling.core.custom_types import AttributesHandler as cls from scrapling.core.custom_types import AttributesHandler as cls
return cls return cls
elif name == 'TextHandler': elif name == "TextHandler":
from scrapling.core.custom_types import TextHandler as cls from scrapling.core.custom_types import TextHandler as cls
return cls return cls
elif name == 'AsyncFetcher': elif name == "AsyncFetcher":
from scrapling.fetchers import AsyncFetcher as cls from scrapling.fetchers import AsyncFetcher as cls
return cls return cls
elif name == 'StealthyFetcher': elif name == "StealthyFetcher":
from scrapling.fetchers import StealthyFetcher as cls from scrapling.fetchers import StealthyFetcher as cls
return cls return cls
elif name == 'PlayWrightFetcher': elif name == "PlayWrightFetcher":
from scrapling.fetchers import PlayWrightFetcher as cls from scrapling.fetchers import PlayWrightFetcher as cls
return cls return cls
elif name == 'CustomFetcher': elif name == "CustomFetcher":
from scrapling.fetchers import CustomFetcher as cls from scrapling.fetchers import CustomFetcher as cls
return cls return cls
else: else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'") raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher'] __all__ = ["Adaptor", "Fetcher", "AsyncFetcher", "StealthyFetcher", "PlayWrightFetcher"]
scrapling/cli.py
+27 -7
View File
@@ -12,21 +12,41 @@ def get_package_dir():
def run_command(command, line): def run_command(command, line):
print(f"Installing {line}...") print(f"Installing {line}...")
_ = subprocess.check_call(' '.join(command), shell=True) _ = subprocess.check_call(" ".join(command), shell=True)
# I meant to not use try except here # I meant to not use try except here
@click.command(help="Install all Scrapling's Fetchers dependencies") @click.command(help="Install all Scrapling's Fetchers dependencies")
@click.option('-f', '--force', 'force', is_flag=True, default=False, type=bool, help="Force Scrapling to reinstall all Fetchers dependencies") @click.option(
"-f",
"--force",
"force",
is_flag=True,
default=False,
type=bool,
help="Force Scrapling to reinstall all Fetchers dependencies",
)
def install(force): def install(force):
if force or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists(): if (
run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers') force
run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies') or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists()
run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases') ):
run_command(
[sys.executable, "-m", "playwright", "install", "chromium"],
"Playwright browsers",
)
run_command(
[sys.executable, "-m", "playwright", "install-deps", "chromium", "firefox"],
"Playwright dependencies",
)
run_command(
[sys.executable, "-m", "camoufox", "fetch", "--browserforge"],
"Camoufox browser and databases",
)
# if no errors raised by above commands, then we add below file # if no errors raised by above commands, then we add below file
get_package_dir().joinpath(".scrapling_dependencies_installed").touch() get_package_dir().joinpath(".scrapling_dependencies_installed").touch()
else: else:
print('The dependencies are already installed') print("The dependencies are already installed")
@click.group() @click.group()
scrapling/core/_types.py
+16 -3
View File
@@ -2,9 +2,22 @@
Type definitions for type checking purposes. Type definitions for type checking purposes.
""" """
from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, from typing import (
List, Literal, Optional, Pattern, Tuple, Type, TypeVar, TYPE_CHECKING,
Union) Any,
Callable,
Dict,
Generator,
Iterable,
List,
Literal,
Optional,
Pattern,
Tuple,
Type,
TypeVar,
Union,
)
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
scrapling/core/custom_types.py
+122 -55
View File
@@ -6,16 +6,26 @@ from types import MappingProxyType
from orjson import dumps, loads from orjson import dumps, loads
from w3lib.html import replace_entities as _replace_entities from w3lib.html import replace_entities as _replace_entities
from scrapling.core._types import (Dict, Iterable, List, Literal, Optional, from scrapling.core._types import (
Pattern, SupportsIndex, TypeVar, Union) Dict,
Iterable,
List,
Literal,
Optional,
Pattern,
SupportsIndex,
TypeVar,
Union,
)
from scrapling.core.utils import _is_iterable, flatten from scrapling.core.utils import _is_iterable, flatten
# Define type variable for AttributeHandler value type # Define type variable for AttributeHandler value type
_TextHandlerType = TypeVar('_TextHandlerType', bound='TextHandler') _TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
class TextHandler(str): class TextHandler(str):
"""Extends standard Python string by adding more functionality""" """Extends standard Python string by adding more functionality"""
__slots__ = () __slots__ = ()
def __new__(cls, string): def __new__(cls, string):
@@ -25,77 +35,89 @@ class TextHandler(str):
lst = super().__getitem__(key) lst = super().__getitem__(key)
return typing.cast(_TextHandlerType, TextHandler(lst)) return typing.cast(_TextHandlerType, TextHandler(lst))
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers': def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers":
return TextHandlers( return TextHandlers(
typing.cast(List[_TextHandlerType], [TextHandler(s) for s in super().split(sep, maxsplit)]) typing.cast(
List[_TextHandlerType],
[TextHandler(s) for s in super().split(sep, maxsplit)],
)
) )
def strip(self, chars: str = None) -> Union[str, 'TextHandler']: def strip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().strip(chars)) return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']: def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().lstrip(chars)) return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']: def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().rstrip(chars)) return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, 'TextHandler']: def capitalize(self) -> Union[str, "TextHandler"]:
return TextHandler(super().capitalize()) return TextHandler(super().capitalize())
def casefold(self) -> Union[str, 'TextHandler']: def casefold(self) -> Union[str, "TextHandler"]:
return TextHandler(super().casefold()) return TextHandler(super().casefold())
def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: def center(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().center(width, fillchar)) return TextHandler(super().center(width, fillchar))
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']: def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]:
return TextHandler(super().expandtabs(tabsize)) return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']: def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]:
return TextHandler(super().format(*args, **kwargs)) return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, 'TextHandler']: def format_map(self, mapping) -> Union[str, "TextHandler"]:
return TextHandler(super().format_map(mapping)) return TextHandler(super().format_map(mapping))
def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']: def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]:
return TextHandler(super().join(iterable)) return TextHandler(super().join(iterable))
def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: def ljust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().ljust(width, fillchar)) return TextHandler(super().ljust(width, fillchar))
def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: def rjust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().rjust(width, fillchar)) return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, 'TextHandler']: def swapcase(self) -> Union[str, "TextHandler"]:
return TextHandler(super().swapcase()) return TextHandler(super().swapcase())
def title(self) -> Union[str, 'TextHandler']: def title(self) -> Union[str, "TextHandler"]:
return TextHandler(super().title()) return TextHandler(super().title())
def translate(self, table) -> Union[str, 'TextHandler']: def translate(self, table) -> Union[str, "TextHandler"]:
return TextHandler(super().translate(table)) return TextHandler(super().translate(table))
def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']: def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]:
return TextHandler(super().zfill(width)) return TextHandler(super().zfill(width))
def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']: def replace(
self, old: str, new: str, count: SupportsIndex = -1
) -> Union[str, "TextHandler"]:
return TextHandler(super().replace(old, new, count)) return TextHandler(super().replace(old, new, count))
def upper(self) -> Union[str, 'TextHandler']: def upper(self) -> Union[str, "TextHandler"]:
return TextHandler(super().upper()) return TextHandler(super().upper())
def lower(self) -> Union[str, 'TextHandler']: def lower(self) -> Union[str, "TextHandler"]:
return TextHandler(super().lower()) return TextHandler(super().lower())
############## ##############
def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']: def sort(self, reverse: bool = False) -> Union[str, "TextHandler"]:
"""Return a sorted version of the string""" """Return a sorted version of the string"""
return self.__class__("".join(sorted(self, reverse=reverse))) return self.__class__("".join(sorted(self, reverse=reverse)))
def clean(self) -> Union[str, 'TextHandler']: def clean(self) -> Union[str, "TextHandler"]:
"""Return a new version of the string after removing all white spaces and consecutive spaces""" """Return a new version of the string after removing all white spaces and consecutive spaces"""
data = re.sub(r'[\t|\r|\n]', '', self) data = re.sub(r"[\t|\r|\n]", "", self)
data = re.sub(' +', ' ', data) data = re.sub(" +", " ", data)
return self.__class__(data.strip()) return self.__class__(data.strip())
# For easy copy-paste from Scrapy/parsel code when needed :) # For easy copy-paste from Scrapy/parsel code when needed :)
@@ -122,8 +144,7 @@ class TextHandler(str):
replace_entities: bool = True, replace_entities: bool = True,
clean_match: bool = False, clean_match: bool = False,
case_sensitive: bool = True, case_sensitive: bool = True,
) -> bool: ) -> bool: ...
...
@typing.overload @typing.overload
def re( def re(
@@ -133,12 +154,15 @@ class TextHandler(str):
clean_match: bool = False, clean_match: bool = False,
case_sensitive: bool = True, case_sensitive: bool = True,
check_match: Literal[False] = False, check_match: Literal[False] = False,
) -> "TextHandlers[TextHandler]": ) -> "TextHandlers[TextHandler]": ...
...
def re( def re(
self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, self,
case_sensitive: bool = True, check_match: bool = False regex: Union[str, Pattern[str]],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
check_match: bool = False,
) -> Union["TextHandlers[TextHandler]", bool]: ) -> Union["TextHandlers[TextHandler]", bool]:
"""Apply the given regex to the current text and return a list of strings with the matches. """Apply the given regex to the current text and return a list of strings with the matches.
@@ -164,12 +188,27 @@ class TextHandler(str):
results = flatten(results) results = flatten(results)
if not replace_entities: if not replace_entities:
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(string) for string in results])) return TextHandlers(
typing.cast(
List[_TextHandlerType], [TextHandler(string) for string in results]
)
)
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results])) return TextHandlers(
typing.cast(
List[_TextHandlerType],
[TextHandler(_replace_entities(s)) for s in results],
)
)
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, def re_first(
clean_match: bool = False, case_sensitive: bool = True) -> "TextHandler": self,
regex: Union[str, Pattern[str]],
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandler":
"""Apply the given regex to text and return the first match if found, otherwise return the default value. """Apply the given regex to text and return the first match if found, otherwise return the default value.
:param regex: Can be either a compiled regular expression or a string. :param regex: Can be either a compiled regular expression or a string.
@@ -179,7 +218,12 @@ class TextHandler(str):
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
""" """
result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive) result = self.re(
regex,
replace_entities,
clean_match=clean_match,
case_sensitive=case_sensitive,
)
return result[0] if result else default return result[0] if result else default
@@ -187,6 +231,7 @@ class TextHandlers(List[TextHandler]):
""" """
The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods. The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
""" """
__slots__ = () __slots__ = ()
@typing.overload @typing.overload
@@ -197,15 +242,22 @@ class TextHandlers(List[TextHandler]):
def __getitem__(self, pos: slice) -> "TextHandlers": def __getitem__(self, pos: slice) -> "TextHandlers":
pass pass
def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers"]: def __getitem__(
self, pos: Union[SupportsIndex, slice]
) -> Union[TextHandler, "TextHandlers"]:
lst = super().__getitem__(pos) lst = super().__getitem__(pos)
if isinstance(pos, slice): if isinstance(pos, slice):
lst = [TextHandler(s) for s in lst] lst = [TextHandler(s) for s in lst]
return TextHandlers(typing.cast(List[_TextHandlerType], lst)) return TextHandlers(typing.cast(List[_TextHandlerType], lst))
return typing.cast(_TextHandlerType, TextHandler(lst)) return typing.cast(_TextHandlerType, TextHandler(lst))
def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, def re(
case_sensitive: bool = True) -> 'TextHandlers[TextHandler]': self,
regex: Union[str, Pattern[str]],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandlers[TextHandler]":
"""Call the ``.re()`` method for each element in this list and return """Call the ``.re()`` method for each element in this list and return
their results flattened as TextHandlers. their results flattened as TextHandlers.
@@ -219,8 +271,14 @@ class TextHandlers(List[TextHandler]):
] ]
return TextHandlers(flatten(results)) return TextHandlers(flatten(results))
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, def re_first(
clean_match: bool = False, case_sensitive: bool = True) -> TextHandler: self,
regex: Union[str, Pattern[str]],
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> TextHandler:
"""Call the ``.re_first()`` method for each element in this list and return """Call the ``.re_first()`` method for each element in this list and return
the first result or the default value otherwise. the first result or the default value otherwise.
@@ -251,26 +309,35 @@ class TextHandlers(List[TextHandler]):
class AttributesHandler(Mapping[str, _TextHandlerType]): class AttributesHandler(Mapping[str, _TextHandlerType]):
"""A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities. """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
If standard dictionary is needed, just convert this class to dictionary with `dict` function If standard dictionary is needed, just convert this class to dictionary with `dict` function
""" """
__slots__ = ('_data',)
__slots__ = ("_data",)
def __init__(self, mapping=None, **kwargs): def __init__(self, mapping=None, **kwargs):
mapping = { mapping = (
key: TextHandler(value) if type(value) is str else value {
for key, value in mapping.items() key: TextHandler(value) if type(value) is str else value
} if mapping is not None else {} for key, value in mapping.items()
}
if mapping is not None
else {}
)
if kwargs: if kwargs:
mapping.update({ mapping.update(
key: TextHandler(value) if type(value) is str else value {
for key, value in kwargs.items() key: TextHandler(value) if type(value) is str else value
}) for key, value in kwargs.items()
}
)
# Fastest read-only mapping type # Fastest read-only mapping type
self._data = MappingProxyType(mapping) self._data = MappingProxyType(mapping)
def get(self, key: str, default: Optional[str] = None) -> Union[_TextHandlerType, None]: def get(
self, key: str, default: Optional[str] = None
) -> Union[_TextHandlerType, None]:
"""Acts like standard dictionary `.get()` method""" """Acts like standard dictionary `.get()` method"""
return self._data.get(key, default) return self._data.get(key, default)
scrapling/core/mixins.py
+20 -16
View File
@@ -1,32 +1,33 @@
class SelectorsGeneration: class SelectorsGeneration:
"""Selectors generation functions """Selectors generation functions
Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591""" Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
def __general_selection(self, selection: str = 'css', full_path=False) -> str: def __general_selection(self, selection: str = "css", full_path=False) -> str:
"""Generate a selector for the current element. """Generate a selector for the current element.
:return: A string of the generated selector. :return: A string of the generated selector.
""" """
selectorPath = [] selectorPath = []
target = self target = self
css = selection.lower() == 'css' css = selection.lower() == "css"
while target is not None: while target is not None:
if target.parent: if target.parent:
if target.attrib.get('id'): if target.attrib.get("id"):
# id is enough # id is enough
part = ( part = (
f'#{target.attrib["id"]}' if css f"#{target.attrib['id']}"
if css
else f"[@id='{target.attrib['id']}']" else f"[@id='{target.attrib['id']}']"
) )
selectorPath.append(part) selectorPath.append(part)
if not full_path: if not full_path:
return ( return (
" > ".join(reversed(selectorPath)) if css " > ".join(reversed(selectorPath))
else '//*' + "/".join(reversed(selectorPath)) if css
else "//*" + "/".join(reversed(selectorPath))
) )
else: else:
part = f'{target.tag}' part = f"{target.tag}"
# We won't use classes anymore because I some websites share exact classes between elements # We won't use classes anymore because I some websites share exact classes between elements
# classes = target.attrib.get('class', '').split() # classes = target.attrib.get('class', '').split()
# if classes and css: # if classes and css:
@@ -41,23 +42,26 @@ class SelectorsGeneration:
if counter[target.tag] > 1: if counter[target.tag] > 1:
part += ( part += (
f":nth-of-type({counter[target.tag]})" if css f":nth-of-type({counter[target.tag]})"
if css
else f"[{counter[target.tag]}]" else f"[{counter[target.tag]}]"
) )
selectorPath.append(part) selectorPath.append(part)
target = target.parent target = target.parent
if target is None or target.tag == 'html': if target is None or target.tag == "html":
return ( return (
" > ".join(reversed(selectorPath)) if css " > ".join(reversed(selectorPath))
else '//' + "/".join(reversed(selectorPath)) if css
else "//" + "/".join(reversed(selectorPath))
) )
else: else:
break break
return ( return (
" > ".join(reversed(selectorPath)) if css " > ".join(reversed(selectorPath))
else '//' + "/".join(reversed(selectorPath)) if css
else "//" + "/".join(reversed(selectorPath))
) )
@property @property
@@ -79,11 +83,11 @@ class SelectorsGeneration:
"""Generate a XPath selector for the current element """Generate a XPath selector for the current element
:return: A string of the generated selector. :return: A string of the generated selector.
""" """
return self.__general_selection('xpath') return self.__general_selection("xpath")
@property @property
def generate_full_xpath_selector(self) -> str: def generate_full_xpath_selector(self) -> str:
"""Generate a complete XPath selector for the current element """Generate a complete XPath selector for the current element
:return: A string of the generated selector. :return: A string of the generated selector.
""" """
return self.__general_selection('xpath', full_path=True) return self.__general_selection("xpath", full_path=True)
scrapling/core/storage_adaptors.py
+11 -7
View File
@@ -20,7 +20,7 @@ class StorageSystemMixin(ABC):
self.url = url self.url = url
@lru_cache(64, typed=True) @lru_cache(64, typed=True)
def _get_base_url(self, default_value: str = 'default') -> str: def _get_base_url(self, default_value: str = "default") -> str:
if not self.url or type(self.url) is not str: if not self.url or type(self.url) is not str:
return default_value return default_value
@@ -38,7 +38,7 @@ class StorageSystemMixin(ABC):
:param identifier: This is the identifier that will be used to retrieve the element later from the storage. See :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
the docs for more info. the docs for more info.
""" """
raise NotImplementedError('Storage system must implement `save` method') raise NotImplementedError("Storage system must implement `save` method")
@abstractmethod @abstractmethod
def retrieve(self, identifier: str) -> Optional[Dict]: def retrieve(self, identifier: str) -> Optional[Dict]:
@@ -48,7 +48,7 @@ class StorageSystemMixin(ABC):
the docs for more info. the docs for more info.
:return: A dictionary of the unique properties :return: A dictionary of the unique properties
""" """
raise NotImplementedError('Storage system must implement `save` method') raise NotImplementedError("Storage system must implement `save` method")
@staticmethod @staticmethod
@lru_cache(128, typed=True) @lru_cache(128, typed=True)
@@ -57,7 +57,7 @@ class StorageSystemMixin(ABC):
identifier = identifier.lower().strip() identifier = identifier.lower().strip()
if isinstance(identifier, str): if isinstance(identifier, str):
# Hash functions have to take bytes # Hash functions have to take bytes
identifier = identifier.encode('utf-8') identifier = identifier.encode("utf-8")
hash_value = sha256(identifier).hexdigest() hash_value = sha256(identifier).hexdigest()
return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance
@@ -68,6 +68,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
"""The recommended system to use, it's race condition safe and thread safe. """The recommended system to use, it's race condition safe and thread safe.
Mainly built so the library can run in threaded frameworks like scrapy or threaded tools Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
> It's optimized for threaded applications but running it without threads shouldn't make it slow.""" > It's optimized for threaded applications but running it without threads shouldn't make it slow."""
def __init__(self, storage_file: str, url: Union[str, None] = None): def __init__(self, storage_file: str, url: Union[str, None] = None):
""" """
:param storage_file: File to be used to store elements :param storage_file: File to be used to store elements
@@ -111,10 +112,13 @@ class SQLiteStorageSystem(StorageSystemMixin):
url = self._get_base_url() url = self._get_base_url()
element_data = _StorageTools.element_to_dict(element) element_data = _StorageTools.element_to_dict(element)
with self.lock: with self.lock:
self.cursor.execute(""" self.cursor.execute(
"""
INSERT OR REPLACE INTO storage (url, identifier, element_data) INSERT OR REPLACE INTO storage (url, identifier, element_data)
VALUES (?, ?, ?) VALUES (?, ?, ?)
""", (url, identifier, orjson.dumps(element_data))) """,
(url, identifier, orjson.dumps(element_data)),
)
self.cursor.fetchall() self.cursor.fetchall()
self.connection.commit() self.connection.commit()
@@ -129,7 +133,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
with self.lock: with self.lock:
self.cursor.execute( self.cursor.execute(
"SELECT element_data FROM storage WHERE url = ? AND identifier = ?", "SELECT element_data FROM storage WHERE url = ? AND identifier = ?",
(url, identifier) (url, identifier),
) )
result = self.cursor.fetchone() result = self.cursor.fetchone()
if result: if result:
scrapling/core/translator.py
+1 -2
View File
@@ -24,7 +24,6 @@ replace_html5_whitespaces = re.compile(regex).sub
class XPathExpr(OriginalXPathExpr): class XPathExpr(OriginalXPathExpr):
textnode: bool = False textnode: bool = False
attribute: Optional[str] = None attribute: Optional[str] = None
@@ -123,7 +122,7 @@ class TranslatorMixin:
@staticmethod @staticmethod
def xpath_attr_functional_pseudo_element( def xpath_attr_functional_pseudo_element(
xpath: OriginalXPathExpr, function: FunctionalPseudoElement xpath: OriginalXPathExpr, function: FunctionalPseudoElement
) -> XPathExpr: ) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element""" """Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]): if function.argument_types() not in (["STRING"], ["IDENT"]):
scrapling/core/utils.py
+44 -25
View File
@@ -11,7 +11,9 @@ from scrapling.core._types import Any, Dict, Iterable, Union
# functools.cache is available on Python 3.9+ only so let's keep lru_cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache
from functools import lru_cache # isort:skip from functools import lru_cache # isort:skip
html_forbidden = {html.HtmlComment, } html_forbidden = {
html.HtmlComment,
}
@lru_cache(1, typed=True) @lru_cache(1, typed=True)
@@ -20,12 +22,11 @@ def setup_logger():
:returns: logging.Logger: Configured logger instance :returns: logging.Logger: Configured logger instance
""" """
logger = logging.getLogger('scrapling') logger = logging.getLogger("scrapling")
logger.setLevel(logging.INFO) logger.setLevel(logging.INFO)
formatter = logging.Formatter( formatter = logging.Formatter(
fmt="[%(asctime)s] %(levelname)s: %(message)s", fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
datefmt="%Y-%m-%d %H:%M:%S"
) )
console_handler = logging.StreamHandler() console_handler = logging.StreamHandler()
@@ -58,7 +59,13 @@ def flatten(lst: Iterable):
def _is_iterable(s: Any): def _is_iterable(s: Any):
# This will be used only in regex functions to make sure it's iterable but not string/bytes # This will be used only in regex functions to make sure it's iterable but not string/bytes
return isinstance(s, (list, tuple,)) return isinstance(
s,
(
list,
tuple,
),
)
class _StorageTools: class _StorageTools:
@@ -66,31 +73,43 @@ class _StorageTools:
def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict: def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:
if not element.attrib: if not element.attrib:
return {} return {}
return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden} return {
k: v.strip()
for k, v in element.attrib.items()
if v and v.strip() and k not in forbidden
}
@classmethod @classmethod
def element_to_dict(cls, element: html.HtmlElement) -> Dict: def element_to_dict(cls, element: html.HtmlElement) -> Dict:
parent = element.getparent() parent = element.getparent()
result = { result = {
'tag': str(element.tag), "tag": str(element.tag),
'attributes': cls.__clean_attributes(element), "attributes": cls.__clean_attributes(element),
'text': element.text.strip() if element.text else None, "text": element.text.strip() if element.text else None,
'path': cls._get_element_path(element) "path": cls._get_element_path(element),
} }
if parent is not None: if parent is not None:
result.update({ result.update(
'parent_name': parent.tag, {
'parent_attribs': dict(parent.attrib), "parent_name": parent.tag,
'parent_text': parent.text.strip() if parent.text else None "parent_attribs": dict(parent.attrib),
}) "parent_text": parent.text.strip() if parent.text else None,
}
)
siblings = [child.tag for child in parent.iterchildren() if child != element] siblings = [
child.tag for child in parent.iterchildren() if child != element
]
if siblings: if siblings:
result.update({'siblings': tuple(siblings)}) result.update({"siblings": tuple(siblings)})
children = [child.tag for child in element.iterchildren() if type(child) not in html_forbidden] children = [
child.tag
for child in element.iterchildren()
if type(child) not in html_forbidden
]
if children: if children:
result.update({'children': tuple(children)}) result.update({"children": tuple(children)})
return result return result
@@ -98,9 +117,9 @@ class _StorageTools:
def _get_element_path(cls, element: html.HtmlElement): def _get_element_path(cls, element: html.HtmlElement):
parent = element.getparent() parent = element.getparent()
return tuple( return tuple(
(element.tag,) if parent is None else ( (element.tag,)
cls._get_element_path(parent) + (element.tag,) if parent is None
) else (cls._get_element_path(parent) + (element.tag,))
) )
@@ -117,6 +136,6 @@ class _StorageTools:
@lru_cache(128, typed=True) @lru_cache(128, typed=True)
def clean_spaces(string): def clean_spaces(string):
string = string.replace('\t', ' ') string = string.replace("\t", " ")
string = re.sub('[\n|\r]', '', string) string = re.sub("[\n|\r]", "", string)
return re.sub(' +', ' ', string) return re.sub(" +", " ", string)
scrapling/defaults.py
+20 -8
View File
@@ -5,21 +5,33 @@ from scrapling.core.utils import log
# A lightweight approach to create lazy loader for each import for backward compatibility # A lightweight approach to create lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used) # This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name): def __getattr__(name):
if name == 'Fetcher': if name == "Fetcher":
from scrapling.fetchers import Fetcher as cls from scrapling.fetchers import Fetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import Fetcher` instead')
log.warning(
"This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import Fetcher` instead"
)
return cls return cls
elif name == 'AsyncFetcher': elif name == "AsyncFetcher":
from scrapling.fetchers import AsyncFetcher as cls from scrapling.fetchers import AsyncFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import AsyncFetcher` instead')
log.warning(
"This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import AsyncFetcher` instead"
)
return cls return cls
elif name == 'StealthyFetcher': elif name == "StealthyFetcher":
from scrapling.fetchers import StealthyFetcher as cls from scrapling.fetchers import StealthyFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import StealthyFetcher` instead')
log.warning(
"This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import StealthyFetcher` instead"
)
return cls return cls
elif name == 'PlayWrightFetcher': elif name == "PlayWrightFetcher":
from scrapling.fetchers import PlayWrightFetcher as cls from scrapling.fetchers import PlayWrightFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import PlayWrightFetcher` instead')
log.warning(
"This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import PlayWrightFetcher` instead"
)
return cls return cls
else: else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'") raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
scrapling/engines/__init__.py
+1 -1
View File
@@ -4,4 +4,4 @@ from .pw import PlaywrightEngine
from .static import StaticEngine from .static import StaticEngine
from .toolbelt import check_if_engine_usable from .toolbelt import check_if_engine_usable
__all__ = ['CamoufoxEngine', 'PlaywrightEngine'] __all__ = ["CamoufoxEngine", "PlaywrightEngine"]
scrapling/engines/camo.py
+125 -59
View File
@@ -2,27 +2,52 @@ from camoufox import DefaultAddons
from camoufox.async_api import AsyncCamoufox from camoufox.async_api import AsyncCamoufox
from camoufox.sync_api import Camoufox from camoufox.sync_api import Camoufox
from scrapling.core._types import (Callable, Dict, List, Literal, Optional, from scrapling.core._types import (
SelectorWaitStates, Union) Callable,
Dict,
List,
Literal,
Optional,
SelectorWaitStates,
Union,
)
from scrapling.core.utils import log from scrapling.core.utils import log
from scrapling.engines.toolbelt import (Response, StatusText, from scrapling.engines.toolbelt import (
async_intercept_route, Response,
check_type_validity, StatusText,
construct_proxy_dict, async_intercept_route,
generate_convincing_referer, check_type_validity,
get_os_name, intercept_route) construct_proxy_dict,
generate_convincing_referer,
get_os_name,
intercept_route,
)
class CamoufoxEngine: class CamoufoxEngine:
def __init__( def __init__(
self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, self,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0, headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None, block_images: bool = False,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, disable_resources: bool = False,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, block_webrtc: bool = False,
geoip: bool = False, allow_webgl: bool = True,
adaptor_arguments: Dict = None, network_idle: bool = False,
additional_arguments: Dict = None humanize: Union[bool, float] = True,
wait: Optional[int] = 0,
timeout: Optional[float] = 30000,
page_action: Callable = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Dict = None,
additional_arguments: Dict = None,
): ):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation. """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -97,7 +122,7 @@ class CamoufoxEngine:
"block_webrtc": self.block_webrtc, "block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful "block_images": self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
"os": None if self.os_randomize else get_os_name(), "os": None if self.os_randomize else get_os_name(),
**self.additional_arguments **self.additional_arguments,
} }
def _process_response_history(self, first_response): def _process_response_history(self, first_response):
@@ -109,19 +134,30 @@ class CamoufoxEngine:
while current_request: while current_request:
try: try:
current_response = current_request.response() current_response = current_request.response()
history.insert(0, Response( history.insert(
url=current_request.url, 0,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" Response(
text='', url=current_request.url,
body=b'', # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
status=current_response.status if current_response else 301, text="",
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), body=b"",
encoding=current_response.headers.get('content-type', '') or 'utf-8', status=current_response.status if current_response else 301,
cookies={}, reason=(
headers=current_response.all_headers() if current_response else {}, current_response.status_text
request_headers=current_request.all_headers(), or StatusText.get(current_response.status)
**self.adaptor_arguments )
)) if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=current_response.all_headers()
if current_response
else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e: except Exception as e:
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
@@ -141,19 +177,30 @@ class CamoufoxEngine:
while current_request: while current_request:
try: try:
current_response = await current_request.response() current_response = await current_request.response()
history.insert(0, Response( history.insert(
url=current_request.url, 0,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" Response(
text='', url=current_request.url,
body=b'', # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
status=current_response.status if current_response else 301, text="",
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), body=b"",
encoding=current_response.headers.get('content-type', '') or 'utf-8', status=current_response.status if current_response else 301,
cookies={}, reason=(
headers=await current_response.all_headers() if current_response else {}, current_response.status_text
request_headers=await current_request.all_headers(), or StatusText.get(current_response.status)
**self.adaptor_arguments )
)) if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=await current_response.all_headers()
if current_response
else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e: except Exception as e:
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
@@ -175,7 +222,10 @@ class CamoufoxEngine:
def handle_response(finished_response): def handle_response(finished_response):
nonlocal final_response nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request(): if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response final_response = finished_response
with Camoufox(**self._get_camoufox_options()) as browser: with Camoufox(**self._get_camoufox_options()) as browser:
@@ -195,7 +245,7 @@ class CamoufoxEngine:
page.wait_for_load_state(state="domcontentloaded") page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
page.wait_for_load_state('networkidle') page.wait_for_load_state("networkidle")
if self.page_action is not None: if self.page_action is not None:
try: try:
@@ -211,7 +261,7 @@ class CamoufoxEngine:
page.wait_for_load_state(state="load") page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded") page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
page.wait_for_load_state('networkidle') page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -222,9 +272,13 @@ class CamoufoxEngine:
raise ValueError("Failed to get a response from the page") raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = self._process_response_history(first_response) history = self._process_response_history(first_response)
try: try:
@@ -236,15 +290,17 @@ class CamoufoxEngine:
response = Response( response = Response(
url=page.url, url=page.url,
text=page_content, text=page_content,
body=page_content.encode('utf-8'), body=page_content.encode("utf-8"),
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, cookies={
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(), headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(), request_headers=first_response.request.all_headers(),
history=history, history=history,
**self.adaptor_arguments **self.adaptor_arguments,
) )
page.close() page.close()
context.close() context.close()
@@ -262,7 +318,10 @@ class CamoufoxEngine:
async def handle_response(finished_response): async def handle_response(finished_response):
nonlocal final_response nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request(): if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response final_response = finished_response
async with AsyncCamoufox(**self._get_camoufox_options()) as browser: async with AsyncCamoufox(**self._get_camoufox_options()) as browser:
@@ -282,7 +341,7 @@ class CamoufoxEngine:
await page.wait_for_load_state(state="domcontentloaded") await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
await page.wait_for_load_state('networkidle') await page.wait_for_load_state("networkidle")
if self.page_action is not None: if self.page_action is not None:
try: try:
@@ -298,7 +357,7 @@ class CamoufoxEngine:
await page.wait_for_load_state(state="load") await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded") await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
await page.wait_for_load_state('networkidle') await page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -309,9 +368,13 @@ class CamoufoxEngine:
raise ValueError("Failed to get a response from the page") raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = await self._async_process_response_history(first_response) history = await self._async_process_response_history(first_response)
try: try:
@@ -323,15 +386,18 @@ class CamoufoxEngine:
response = Response( response = Response(
url=page.url, url=page.url,
text=page_content, text=page_content,
body=page_content.encode('utf-8'), body=page_content.encode("utf-8"),
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()}, cookies={
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(), headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(), request_headers=await first_response.request.all_headers(),
history=history, history=history,
**self.adaptor_arguments **self.adaptor_arguments,
) )
await page.close() await page.close()
await context.close() await context.close()
scrapling/engines/constants.py
+84 -87
View File
@@ -1,92 +1,92 @@
# Disable loading these resources for speed # Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = { DEFAULT_DISABLED_RESOURCES = {
'font', "font",
'image', "image",
'media', "media",
'beacon', "beacon",
'object', "object",
'imageset', "imageset",
'texttrack', "texttrack",
'websocket', "websocket",
'csp_report', "csp_report",
'stylesheet', "stylesheet",
} }
DEFAULT_STEALTH_FLAGS = ( DEFAULT_STEALTH_FLAGS = (
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/ # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable # Generally this will make the browser faster and less detectable
'--no-pings', "--no-pings",
'--incognito', "--incognito",
'--test-type', "--test-type",
'--lang=en-US', "--lang=en-US",
'--mute-audio', "--mute-audio",
'--no-first-run', "--no-first-run",
'--disable-sync', "--disable-sync",
'--hide-scrollbars', "--hide-scrollbars",
'--disable-logging', "--disable-logging",
'--start-maximized', # For headless check bypass "--start-maximized", # For headless check bypass
'--enable-async-dns', "--enable-async-dns",
'--disable-breakpad', "--disable-breakpad",
'--disable-infobars', "--disable-infobars",
'--accept-lang=en-US', "--accept-lang=en-US",
'--use-mock-keychain', "--use-mock-keychain",
'--disable-translate', "--disable-translate",
'--disable-extensions', "--disable-extensions",
'--disable-voice-input', "--disable-voice-input",
'--window-position=0,0', "--window-position=0,0",
'--disable-wake-on-wifi', "--disable-wake-on-wifi",
'--ignore-gpu-blocklist', "--ignore-gpu-blocklist",
'--enable-tcp-fast-open', "--enable-tcp-fast-open",
'--enable-web-bluetooth', "--enable-web-bluetooth",
'--disable-hang-monitor', "--disable-hang-monitor",
'--password-store=basic', "--password-store=basic",
'--disable-cloud-import', "--disable-cloud-import",
'--disable-default-apps', "--disable-default-apps",
'--disable-print-preview', "--disable-print-preview",
'--disable-dev-shm-usage', "--disable-dev-shm-usage",
# '--disable-popup-blocking', # '--disable-popup-blocking',
'--metrics-recording-only', "--metrics-recording-only",
'--disable-crash-reporter', "--disable-crash-reporter",
'--disable-partial-raster', "--disable-partial-raster",
'--disable-gesture-typing', "--disable-gesture-typing",
'--disable-checker-imaging', "--disable-checker-imaging",
'--disable-prompt-on-repost', "--disable-prompt-on-repost",
'--force-color-profile=srgb', "--force-color-profile=srgb",
'--font-render-hinting=none', "--font-render-hinting=none",
'--no-default-browser-check', "--no-default-browser-check",
'--aggressive-cache-discard', "--aggressive-cache-discard",
'--disable-component-update', "--disable-component-update",
'--disable-cookie-encryption', "--disable-cookie-encryption",
'--disable-domain-reliability', "--disable-domain-reliability",
'--disable-threaded-animation', "--disable-threaded-animation",
'--disable-threaded-scrolling', "--disable-threaded-scrolling",
# '--disable-reading-from-canvas', # For Firefox # '--disable-reading-from-canvas', # For Firefox
'--enable-simple-cache-backend', "--enable-simple-cache-backend",
'--disable-background-networking', "--disable-background-networking",
'--disable-session-crashed-bubble', "--disable-session-crashed-bubble",
'--enable-surface-synchronization', "--enable-surface-synchronization",
'--disable-image-animation-resync', "--disable-image-animation-resync",
'--disable-renderer-backgrounding', "--disable-renderer-backgrounding",
'--disable-ipc-flooding-protection', "--disable-ipc-flooding-protection",
'--prerender-from-omnibox=disabled', "--prerender-from-omnibox=disabled",
'--safebrowsing-disable-auto-update', "--safebrowsing-disable-auto-update",
'--disable-offer-upload-credit-cards', "--disable-offer-upload-credit-cards",
'--disable-features=site-per-process', "--disable-features=site-per-process",
'--disable-background-timer-throttling', "--disable-background-timer-throttling",
'--disable-new-content-rendering-timeout', "--disable-new-content-rendering-timeout",
'--run-all-compositor-stages-before-draw', "--run-all-compositor-stages-before-draw",
'--disable-client-side-phishing-detection', "--disable-client-side-phishing-detection",
'--disable-backgrounding-occluded-windows', "--disable-backgrounding-occluded-windows",
'--disable-layer-tree-host-memory-pressure', "--disable-layer-tree-host-memory-pressure",
'--autoplay-policy=no-user-gesture-required', "--autoplay-policy=no-user-gesture-required",
'--disable-offer-store-unmasked-wallet-cards', "--disable-offer-store-unmasked-wallet-cards",
'--disable-blink-features=AutomationControlled', "--disable-blink-features=AutomationControlled",
'--webrtc-ip-handling-policy=disable_non_proxied_udp', "--webrtc-ip-handling-policy=disable_non_proxied_udp",
'--disable-component-extensions-with-background-pages', "--disable-component-extensions-with-background-pages",
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp', "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance', "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance",
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4', "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees', "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
) )
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
@@ -95,13 +95,10 @@ NSTBROWSER_DEFAULT_QUERY = {
"headless": True, "headless": True,
"autoClose": True, "autoClose": True,
"fingerprint": { "fingerprint": {
"flags": { "flags": {"timezone": "BasedOnIp", "screen": "Custom"},
"timezone": "BasedOnIp", "platform": "linux", # support: windows, mac, linux
"screen": "Custom" "kernel": "chromium", # only support: chromium
}, "kernelMilestone": "128",
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"hardwareConcurrency": 8, "hardwareConcurrency": 8,
"deviceMemory": 8, "deviceMemory": 8,
}, },
scrapling/engines/pw.py
+169 -100
View File
@@ -1,42 +1,46 @@
import json import json
from scrapling.core._types import (Callable, Dict, Optional, from scrapling.core._types import Callable, Dict, Optional, SelectorWaitStates, Union
SelectorWaitStates, Union)
from scrapling.core.utils import log, lru_cache from scrapling.core.utils import log, lru_cache
from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS, from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
NSTBROWSER_DEFAULT_QUERY) from scrapling.engines.toolbelt import (
from scrapling.engines.toolbelt import (Response, StatusText, Response,
async_intercept_route, StatusText,
check_type_validity, construct_cdp_url, async_intercept_route,
construct_proxy_dict, check_type_validity,
generate_convincing_referer, construct_cdp_url,
generate_headers, intercept_route, construct_proxy_dict,
js_bypass_path) generate_convincing_referer,
generate_headers,
intercept_route,
js_bypass_path,
)
class PlaywrightEngine: class PlaywrightEngine:
def __init__( def __init__(
self, headless: Union[bool, str] = True, self,
disable_resources: bool = False, headless: Union[bool, str] = True,
useragent: Optional[str] = None, disable_resources: bool = False,
network_idle: bool = False, useragent: Optional[str] = None,
timeout: Optional[float] = 30000, network_idle: bool = False,
wait: Optional[int] = 0, timeout: Optional[float] = 30000,
page_action: Callable = None, wait: Optional[int] = 0,
wait_selector: Optional[str] = None, page_action: Callable = None,
locale: Optional[str] = 'en-US', wait_selector: Optional[str] = None,
wait_selector_state: SelectorWaitStates = 'attached', locale: Optional[str] = "en-US",
stealth: bool = False, wait_selector_state: SelectorWaitStates = "attached",
real_chrome: bool = False, stealth: bool = False,
hide_canvas: bool = False, real_chrome: bool = False,
disable_webgl: bool = False, hide_canvas: bool = False,
cdp_url: Optional[str] = None, disable_webgl: bool = False,
nstbrowser_mode: bool = False, cdp_url: Optional[str] = None,
nstbrowser_config: Optional[Dict] = None, nstbrowser_mode: bool = False,
google_search: bool = True, nstbrowser_config: Optional[Dict] = None,
extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None,
adaptor_arguments: Dict = None proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None,
): ):
"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation. """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
@@ -65,7 +69,7 @@ class PlaywrightEngine:
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
""" """
self.headless = headless self.headless = headless
self.locale = check_type_validity(locale, [str], 'en-US', param_name='locale') self.locale = check_type_validity(locale, [str], "en-US", param_name="locale")
self.disable_resources = disable_resources self.disable_resources = disable_resources
self.network_idle = bool(network_idle) self.network_idle = bool(network_idle)
self.stealth = bool(stealth) self.stealth = bool(stealth)
@@ -95,8 +99,8 @@ class PlaywrightEngine:
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {} self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
self.harmful_default_args = [ self.harmful_default_args = [
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884 # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
'--enable-automation', "--enable-automation",
'--disable-popup-blocking', "--disable-popup-blocking",
# '--disable-component-update', # '--disable-component-update',
# '--disable-default-apps', # '--disable-default-apps',
# '--disable-extensions', # '--disable-extensions',
@@ -114,12 +118,16 @@ class PlaywrightEngine:
query = NSTBROWSER_DEFAULT_QUERY.copy() query = NSTBROWSER_DEFAULT_QUERY.copy()
if self.stealth: if self.stealth:
flags = self.__set_flags() flags = self.__set_flags()
query.update({ query.update(
"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary {
}) "args": dict(
zip(flags, [""] * len(flags))
), # browser args should be a dictionary
}
)
config = { config = {
'config': json.dumps(query), "config": json.dumps(query),
# 'token': '' # 'token': ''
} }
cdp_url = construct_cdp_url(cdp_url, config) cdp_url = construct_cdp_url(cdp_url, config)
@@ -134,17 +142,25 @@ class PlaywrightEngine:
"""Returns the flags that will be used while launching the browser if stealth mode is enabled""" """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS flags = DEFAULT_STEALTH_FLAGS
if self.hide_canvas: if self.hide_canvas:
flags += ('--fingerprinting-canvas-image-data-noise',) flags += ("--fingerprinting-canvas-image-data-noise",)
if self.disable_webgl: if self.disable_webgl:
flags += ('--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2',) flags += (
"--disable-webgl",
"--disable-webgl-image-chromium",
"--disable-webgl2",
)
return flags return flags
def __launch_kwargs(self): def __launch_kwargs(self):
"""Creates the arguments we will use while launching playwright's browser""" """Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {'headless': self.headless, 'ignore_default_args': self.harmful_default_args, 'channel': 'chrome' if self.real_chrome else 'chromium'} launch_kwargs = {
"headless": self.headless,
"ignore_default_args": self.harmful_default_args,
"channel": "chrome" if self.real_chrome else "chromium",
}
if self.stealth: if self.stealth:
launch_kwargs.update({'args': self.__set_flags(), 'chromium_sandbox': True}) launch_kwargs.update({"args": self.__set_flags(), "chromium_sandbox": True})
return launch_kwargs return launch_kwargs
@@ -153,22 +169,26 @@ class PlaywrightEngine:
context_kwargs = { context_kwargs = {
"proxy": self.proxy, "proxy": self.proxy,
"locale": self.locale, "locale": self.locale,
"color_scheme": 'dark', # Bypasses the 'prefersLightColor' check in creepjs "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
"device_scale_factor": 2, "device_scale_factor": 2,
"extra_http_headers": self.extra_headers if self.extra_headers else {}, "extra_http_headers": self.extra_headers if self.extra_headers else {},
"user_agent": self.useragent if self.useragent else generate_headers(browser_mode=True).get('User-Agent'), "user_agent": self.useragent
if self.useragent
else generate_headers(browser_mode=True).get("User-Agent"),
} }
if self.stealth: if self.stealth:
context_kwargs.update({ context_kwargs.update(
'is_mobile': False, {
'has_touch': False, "is_mobile": False,
# I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now "has_touch": False,
'service_workers': 'allow', # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
'ignore_https_errors': True, "service_workers": "allow",
'screen': {'width': 1920, 'height': 1080}, "ignore_https_errors": True,
'viewport': {'width': 1920, 'height': 1080}, "screen": {"width": 1920, "height": 1080},
'permissions': ['geolocation', 'notifications'] "viewport": {"width": 1920, "height": 1080},
}) "permissions": ["geolocation", "notifications"],
}
)
return context_kwargs return context_kwargs
@@ -184,10 +204,16 @@ class PlaywrightEngine:
# https://arh.antoinevastel.com/bots/areyouheadless/ # https://arh.antoinevastel.com/bots/areyouheadless/
# https://prescience-data.github.io/execution-monitor.html # https://prescience-data.github.io/execution-monitor.html
return tuple( return tuple(
js_bypass_path(script) for script in ( js_bypass_path(script)
for script in (
# Order is important # Order is important
'webdriver_fully.js', 'window_chrome.js', 'navigator_plugins.js', 'pdf_viewer.js', "webdriver_fully.js",
'notification_permission.js', 'screen_props.js', 'playwright_fingerprint.js' "window_chrome.js",
"navigator_plugins.js",
"pdf_viewer.js",
"notification_permission.js",
"screen_props.js",
"playwright_fingerprint.js",
) )
) )
@@ -200,19 +226,30 @@ class PlaywrightEngine:
while current_request: while current_request:
try: try:
current_response = current_request.response() current_response = current_request.response()
history.insert(0, Response( history.insert(
url=current_request.url, 0,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" Response(
text='', url=current_request.url,
body=b'', # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
status=current_response.status if current_response else 301, text="",
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), body=b"",
encoding=current_response.headers.get('content-type', '') or 'utf-8', status=current_response.status if current_response else 301,
cookies={}, reason=(
headers=current_response.all_headers() if current_response else {}, current_response.status_text
request_headers=current_request.all_headers(), or StatusText.get(current_response.status)
**self.adaptor_arguments )
)) if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=current_response.all_headers()
if current_response
else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e: except Exception as e:
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
@@ -232,19 +269,30 @@ class PlaywrightEngine:
while current_request: while current_request:
try: try:
current_response = await current_request.response() current_response = await current_request.response()
history.insert(0, Response( history.insert(
url=current_request.url, 0,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" Response(
text='', url=current_request.url,
body=b'', # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
status=current_response.status if current_response else 301, text="",
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), body=b"",
encoding=current_response.headers.get('content-type', '') or 'utf-8', status=current_response.status if current_response else 301,
cookies={}, reason=(
headers=await current_response.all_headers() if current_response else {}, current_response.status_text
request_headers=await current_request.all_headers(), or StatusText.get(current_response.status)
**self.adaptor_arguments )
)) if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies={},
headers=await current_response.all_headers()
if current_response
else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments,
),
)
except Exception as e: except Exception as e:
log.error(f"Error processing redirect: {e}") log.error(f"Error processing redirect: {e}")
break break
@@ -262,6 +310,7 @@ class PlaywrightEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
from playwright.sync_api import Response as PlaywrightResponse from playwright.sync_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome: if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers # Because rebrowser_playwright doesn't play well with real browsers
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
@@ -273,7 +322,10 @@ class PlaywrightEngine:
def handle_response(finished_response: PlaywrightResponse): def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request(): if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response final_response = finished_response
with sync_playwright() as p: with sync_playwright() as p:
@@ -304,7 +356,7 @@ class PlaywrightEngine:
page.wait_for_load_state(state="domcontentloaded") page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
page.wait_for_load_state('networkidle') page.wait_for_load_state("networkidle")
if self.page_action is not None: if self.page_action is not None:
try: try:
@@ -320,7 +372,7 @@ class PlaywrightEngine:
page.wait_for_load_state(state="load") page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded") page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
page.wait_for_load_state('networkidle') page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -331,9 +383,13 @@ class PlaywrightEngine:
raise ValueError("Failed to get a response from the page") raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = self._process_response_history(first_response) history = self._process_response_history(first_response)
try: try:
@@ -345,15 +401,17 @@ class PlaywrightEngine:
response = Response( response = Response(
url=page.url, url=page.url,
text=page_content, text=page_content,
body=page_content.encode('utf-8'), body=page_content.encode("utf-8"),
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, cookies={
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(), headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(), request_headers=first_response.request.all_headers(),
history=history, history=history,
**self.adaptor_arguments **self.adaptor_arguments,
) )
page.close() page.close()
context.close() context.close()
@@ -366,6 +424,7 @@ class PlaywrightEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
from playwright.async_api import Response as PlaywrightResponse from playwright.async_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome: if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers # Because rebrowser_playwright doesn't play well with real browsers
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
@@ -377,7 +436,10 @@ class PlaywrightEngine:
async def handle_response(finished_response: PlaywrightResponse): async def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request(): if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response final_response = finished_response
async with async_playwright() as p: async with async_playwright() as p:
@@ -408,7 +470,7 @@ class PlaywrightEngine:
await page.wait_for_load_state(state="domcontentloaded") await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
await page.wait_for_load_state('networkidle') await page.wait_for_load_state("networkidle")
if self.page_action is not None: if self.page_action is not None:
try: try:
@@ -424,7 +486,7 @@ class PlaywrightEngine:
await page.wait_for_load_state(state="load") await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded") await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle: if self.network_idle:
await page.wait_for_load_state('networkidle') await page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
@@ -435,9 +497,13 @@ class PlaywrightEngine:
raise ValueError("Failed to get a response from the page") raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = await self._async_process_response_history(first_response) history = await self._async_process_response_history(first_response)
try: try:
@@ -449,15 +515,18 @@ class PlaywrightEngine:
response = Response( response = Response(
url=page.url, url=page.url,
text=page_content, text=page_content,
body=page_content.encode('utf-8'), body=page_content.encode("utf-8"),
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()}, cookies={
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(), headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(), request_headers=await first_response.request.all_headers(),
history=history, history=history,
**self.adaptor_arguments **self.adaptor_arguments,
) )
await page.close() await page.close()
await context.close() await context.close()
scrapling/engines/static.py
+57 -25
View File
@@ -10,8 +10,14 @@ from .toolbelt import Response, generate_convincing_referer, generate_headers
@lru_cache(2, typed=True) # Singleton easily @lru_cache(2, typed=True) # Singleton easily
class StaticEngine: class StaticEngine:
def __init__( def __init__(
self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True, self,
timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None url: str,
proxy: Optional[str] = None,
stealthy_headers: bool = True,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None,
retries: Optional[int] = 3,
adaptor_arguments: Tuple = None,
): ):
"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation. """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
@@ -47,14 +53,22 @@ class StaticEngine:
if self.stealth: if self.stealth:
extra_headers = generate_headers(browser_mode=False) extra_headers = generate_headers(browser_mode=False)
# Don't overwrite user supplied headers # Don't overwrite user supplied headers
extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} extra_headers = {
key: value
for key, value in extra_headers.items()
if key.lower() not in headers_keys
}
headers.update(extra_headers) headers.update(extra_headers)
if 'referer' not in headers_keys: if "referer" not in headers_keys:
headers.update({'referer': generate_convincing_referer(self.url)}) headers.update({"referer": generate_convincing_referer(self.url)})
elif 'user-agent' not in headers_keys: elif "user-agent" not in headers_keys:
headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent') headers["User-Agent"] = generate_headers(browser_mode=False).get(
log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") "User-Agent"
)
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."
)
return headers return headers
@@ -70,25 +84,43 @@ class StaticEngine:
body=response.content, body=response.content,
status=response.status_code, status=response.status_code,
reason=response.reason_phrase, reason=response.reason_phrase,
encoding=response.encoding or 'utf-8', encoding=response.encoding or "utf-8",
cookies=dict(response.cookies), cookies=dict(response.cookies),
headers=dict(response.headers), headers=dict(response.headers),
request_headers=dict(response.request.headers), request_headers=dict(response.request.headers),
method=response.request.method, method=response.request.method,
history=[self._prepare_response(redirection) for redirection in response.history], history=[
**self.adaptor_arguments self._prepare_response(redirection) for redirection in response.history
],
**self.adaptor_arguments,
) )
def _make_request(self, method: str, **kwargs) -> Response: def _make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {})) headers = self._headers_job(kwargs.pop("headers", {}))
with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client: with httpx.Client(
request = getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)
) as client:
request = getattr(client, method)(
url=self.url,
headers=headers,
follow_redirects=self.follow_redirects,
timeout=self.timeout,
**kwargs,
)
return self._prepare_response(request) return self._prepare_response(request)
async def _async_make_request(self, method: str, **kwargs) -> Response: async def _async_make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {})) headers = self._headers_job(kwargs.pop("headers", {}))
async with httpx.AsyncClient(proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)) as client: async with httpx.AsyncClient(
request = await getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)
) as client:
request = await getattr(client, method)(
url=self.url,
headers=headers,
follow_redirects=self.follow_redirects,
timeout=self.timeout,
**kwargs,
)
return self._prepare_response(request) return self._prepare_response(request)
def get(self, **kwargs: Dict) -> Response: def get(self, **kwargs: Dict) -> Response:
@@ -97,7 +129,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return self._make_request('get', **kwargs) return self._make_request("get", **kwargs)
async def async_get(self, **kwargs: Dict) -> Response: async def async_get(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP GET request for you but with some added flavors. """Make basic async HTTP GET request for you but with some added flavors.
@@ -105,7 +137,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return await self._async_make_request('get', **kwargs) return await self._async_make_request("get", **kwargs)
def post(self, **kwargs: Dict) -> Response: def post(self, **kwargs: Dict) -> Response:
"""Make basic HTTP POST request for you but with some added flavors. """Make basic HTTP POST request for you but with some added flavors.
@@ -113,7 +145,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return self._make_request('post', **kwargs) return self._make_request("post", **kwargs)
async def async_post(self, **kwargs: Dict) -> Response: async def async_post(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP POST request for you but with some added flavors. """Make basic async HTTP POST request for you but with some added flavors.
@@ -121,7 +153,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return await self._async_make_request('post', **kwargs) return await self._async_make_request("post", **kwargs)
def delete(self, **kwargs: Dict) -> Response: def delete(self, **kwargs: Dict) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors. """Make basic HTTP DELETE request for you but with some added flavors.
@@ -129,7 +161,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return self._make_request('delete', **kwargs) return self._make_request("delete", **kwargs)
async def async_delete(self, **kwargs: Dict) -> Response: async def async_delete(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP DELETE request for you but with some added flavors. """Make basic async HTTP DELETE request for you but with some added flavors.
@@ -137,7 +169,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return await self._async_make_request('delete', **kwargs) return await self._async_make_request("delete", **kwargs)
def put(self, **kwargs: Dict) -> Response: def put(self, **kwargs: Dict) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors. """Make basic HTTP PUT request for you but with some added flavors.
@@ -145,7 +177,7 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return self._make_request('put', **kwargs) return self._make_request("put", **kwargs)
async def async_put(self, **kwargs: Dict) -> Response: async def async_put(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP PUT request for you but with some added flavors. """Make basic async HTTP PUT request for you but with some added flavors.
@@ -153,4 +185,4 @@ class StaticEngine:
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return await self._async_make_request('put', **kwargs) return await self._async_make_request("put", **kwargs)
scrapling/engines/toolbelt/__init__.py
+16 -6
View File
@@ -1,6 +1,16 @@
from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable, from .custom import (
check_type_validity, get_variable_name) BaseFetcher,
from .fingerprints import (generate_convincing_referer, generate_headers, Response,
get_os_name) StatusText,
from .navigation import (async_intercept_route, construct_cdp_url, check_if_engine_usable,
construct_proxy_dict, intercept_route, js_bypass_path) check_type_validity,
get_variable_name,
)
from .fingerprints import generate_convincing_referer, generate_headers, get_os_name
from .navigation import (
async_intercept_route,
construct_cdp_url,
construct_proxy_dict,
intercept_route,
js_bypass_path,
)
scrapling/engines/toolbelt/custom.py
+167 -95
View File
@@ -1,11 +1,20 @@
""" """
Functions related to custom types or type checking Functions related to custom types or type checking
""" """
import inspect import inspect
from email.message import Message from email.message import Message
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple, from scrapling.core._types import (
Type, Union) Any,
Callable,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
from scrapling.core.custom_types import MappingProxyType from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem from scrapling.parser import Adaptor, SQLiteStorageSystem
@@ -13,7 +22,12 @@ from scrapling.parser import Adaptor, SQLiteStorageSystem
class ResponseEncoding: class ResponseEncoding:
__DEFAULT_ENCODING = "utf-8" __DEFAULT_ENCODING = "utf-8"
__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} __ISO_8859_1_CONTENT_TYPES = {
"text/plain",
"text/html",
"text/css",
"text/javascript",
}
@classmethod @classmethod
@lru_cache(maxsize=128) @lru_cache(maxsize=128)
@@ -27,19 +41,21 @@ class ResponseEncoding:
""" """
# Create a Message object and set the Content-Type header then get the content type and parameters # Create a Message object and set the Content-Type header then get the content type and parameters
msg = Message() msg = Message()
msg['content-type'] = header_value msg["content-type"] = header_value
content_type = msg.get_content_type() content_type = msg.get_content_type()
params = dict(msg.get_params(failobj=[])) params = dict(msg.get_params(failobj=[]))
# Remove the content-type from params if present somehow # Remove the content-type from params if present somehow
params.pop('content-type', None) params.pop("content-type", None)
return content_type, params return content_type, params
@classmethod @classmethod
@lru_cache(maxsize=128) @lru_cache(maxsize=128)
def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str: def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
"""Determine the appropriate character encoding from a content-type header. """Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order: The encoding is determined by these rules in order:
@@ -72,7 +88,9 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING encoding = cls.__DEFAULT_ENCODING
if encoding: if encoding:
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text _ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
return encoding return encoding
return cls.__DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
@@ -84,9 +102,22 @@ class ResponseEncoding:
class Response(Adaptor): class Response(Adaptor):
"""This class is returned by all engines as a way to unify response type between different libraries.""" """This class is returned by all engines as a way to unify response type between different libraries."""
def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, def __init__(
encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict): self,
automatch_domain = adaptor_arguments.pop('automatch_domain', None) url: str,
text: str,
body: bytes,
status: int,
reason: str,
cookies: Dict,
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**adaptor_arguments: Dict,
):
automatch_domain = adaptor_arguments.pop("automatch_domain", None)
self.status = status self.status = status
self.reason = reason self.reason = reason
self.cookies = cookies self.cookies = cookies
@@ -94,11 +125,19 @@ class Response(Adaptor):
self.request_headers = request_headers self.request_headers = request_headers
self.history = history or [] self.history = history or []
encoding = ResponseEncoding.get_value(encoding, text) encoding = ResponseEncoding.get_value(encoding, text)
super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments) super().__init__(
text=text,
body=body,
url=automatch_domain or url,
encoding=encoding,
**adaptor_arguments,
)
# For back-ward compatibility # For back-ward compatibility
self.adaptor = self self.adaptor = self
# For easier debugging while working from a Python shell # For easier debugging while working from a Python shell
log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})') log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
# def __repr__(self): # def __repr__(self):
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>' # return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
@@ -113,16 +152,26 @@ class BaseFetcher:
storage_args: Optional[Dict] = None storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None automatch_domain: Optional[str] = None
parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',) # Left open for the user parser_keywords: Tuple = (
"huge_tree",
"auto_match",
"storage",
"keep_cdata",
"storage_args",
"keep_comments",
"automatch_domain",
) # Left open for the user
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
# For backward-compatibility before 0.2.99 # For backward-compatibility before 0.2.99
args_str = ", ".join(args) or '' args_str = ", ".join(args) or ""
kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or '' kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) or ""
if args_str: if args_str:
args_str += ', ' args_str += ", "
log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching') log.warning(
f"This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching"
)
pass pass
@classmethod @classmethod
@@ -150,12 +199,18 @@ class BaseFetcher:
setattr(cls, key, value) setattr(cls, key, value)
else: else:
# Yup, no fun allowed LOL # Yup, no fun allowed LOL
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
else: else:
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?') raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
if not kwargs: if not kwargs:
raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?') raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
@classmethod @classmethod
def _generate_parser_arguments(cls) -> Dict: def _generate_parser_arguments(cls) -> Dict:
@@ -167,13 +222,15 @@ class BaseFetcher:
keep_cdata=cls.keep_cdata, keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match, auto_match=cls.auto_match,
storage=cls.storage, storage=cls.storage,
storage_args=cls.storage_args storage_args=cls.storage_args,
) )
if cls.automatch_domain: if cls.automatch_domain:
if type(cls.automatch_domain) is not str: if type(cls.automatch_domain) is not str:
log.warning('[Ignored] The argument "automatch_domain" must be of string type') log.warning(
'[Ignored] The argument "automatch_domain" must be of string type'
)
else: else:
parser_arguments.update({'automatch_domain': cls.automatch_domain}) parser_arguments.update({"automatch_domain": cls.automatch_domain})
return parser_arguments return parser_arguments
@@ -181,72 +238,75 @@ class BaseFetcher:
class StatusText: class StatusText:
"""A class that gets the status text of response status code. """A class that gets the status text of response status code.
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
""" """
_phrases = MappingProxyType({
100: "Continue", _phrases = MappingProxyType(
101: "Switching Protocols", {
102: "Processing", 100: "Continue",
103: "Early Hints", 101: "Switching Protocols",
200: "OK", 102: "Processing",
201: "Created", 103: "Early Hints",
202: "Accepted", 200: "OK",
203: "Non-Authoritative Information", 201: "Created",
204: "No Content", 202: "Accepted",
205: "Reset Content", 203: "Non-Authoritative Information",
206: "Partial Content", 204: "No Content",
207: "Multi-Status", 205: "Reset Content",
208: "Already Reported", 206: "Partial Content",
226: "IM Used", 207: "Multi-Status",
300: "Multiple Choices", 208: "Already Reported",
301: "Moved Permanently", 226: "IM Used",
302: "Found", 300: "Multiple Choices",
303: "See Other", 301: "Moved Permanently",
304: "Not Modified", 302: "Found",
305: "Use Proxy", 303: "See Other",
307: "Temporary Redirect", 304: "Not Modified",
308: "Permanent Redirect", 305: "Use Proxy",
400: "Bad Request", 307: "Temporary Redirect",
401: "Unauthorized", 308: "Permanent Redirect",
402: "Payment Required", 400: "Bad Request",
403: "Forbidden", 401: "Unauthorized",
404: "Not Found", 402: "Payment Required",
405: "Method Not Allowed", 403: "Forbidden",
406: "Not Acceptable", 404: "Not Found",
407: "Proxy Authentication Required", 405: "Method Not Allowed",
408: "Request Timeout", 406: "Not Acceptable",
409: "Conflict", 407: "Proxy Authentication Required",
410: "Gone", 408: "Request Timeout",
411: "Length Required", 409: "Conflict",
412: "Precondition Failed", 410: "Gone",
413: "Payload Too Large", 411: "Length Required",
414: "URI Too Long", 412: "Precondition Failed",
415: "Unsupported Media Type", 413: "Payload Too Large",
416: "Range Not Satisfiable", 414: "URI Too Long",
417: "Expectation Failed", 415: "Unsupported Media Type",
418: "I'm a teapot", 416: "Range Not Satisfiable",
421: "Misdirected Request", 417: "Expectation Failed",
422: "Unprocessable Entity", 418: "I'm a teapot",
423: "Locked", 421: "Misdirected Request",
424: "Failed Dependency", 422: "Unprocessable Entity",
425: "Too Early", 423: "Locked",
426: "Upgrade Required", 424: "Failed Dependency",
428: "Precondition Required", 425: "Too Early",
429: "Too Many Requests", 426: "Upgrade Required",
431: "Request Header Fields Too Large", 428: "Precondition Required",
451: "Unavailable For Legal Reasons", 429: "Too Many Requests",
500: "Internal Server Error", 431: "Request Header Fields Too Large",
501: "Not Implemented", 451: "Unavailable For Legal Reasons",
502: "Bad Gateway", 500: "Internal Server Error",
503: "Service Unavailable", 501: "Not Implemented",
504: "Gateway Timeout", 502: "Bad Gateway",
505: "HTTP Version Not Supported", 503: "Service Unavailable",
506: "Variant Also Negotiates", 504: "Gateway Timeout",
507: "Insufficient Storage", 505: "HTTP Version Not Supported",
508: "Loop Detected", 506: "Variant Also Negotiates",
510: "Not Extended", 507: "Insufficient Storage",
511: "Network Authentication Required" 508: "Loop Detected",
}) 510: "Not Extended",
511: "Network Authentication Required",
}
)
@classmethod @classmethod
@lru_cache(maxsize=128) @lru_cache(maxsize=128)
@@ -265,20 +325,26 @@ def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
# if isinstance(engine, type): # if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine") # raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'): if hasattr(engine, "fetch"):
fetch_function = getattr(engine, "fetch") fetch_function = getattr(engine, "fetch")
if callable(fetch_function): if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0: if len(inspect.signature(fetch_function).parameters) > 0:
return engine return engine
else: else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.") # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.") raise TypeError(
"Engine class must have a callable method 'fetch' with the first argument used for the url."
)
else: else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'") # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'") raise TypeError(
"Invalid engine class! Engine class must have a callable method 'fetch'"
)
else: else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'") # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'") raise TypeError(
"Invalid engine class! Engine class must have the method 'fetch'"
)
def get_variable_name(var: Any) -> Optional[str]: def get_variable_name(var: Any) -> Optional[str]:
@@ -293,7 +359,13 @@ def get_variable_name(var: Any) -> Optional[str]:
return None return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any: def check_type_validity(
variable: Any,
valid_types: Union[List[Type], None],
default_value: Any = None,
critical: bool = False,
param_name: Optional[str] = None,
) -> Any:
"""Check if a variable matches the specified type constraints. """Check if a variable matches the specified type constraints.
:param variable: The variable to check :param variable: The variable to check
:param valid_types: List of valid types for the variable :param valid_types: List of valid types for the variable
@@ -316,7 +388,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" cannot be None' error_msg = f'Argument "{var_name}" cannot be None'
if critical: if critical:
raise TypeError(error_msg) raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}') log.error(f"[Ignored] {error_msg}")
return default_value return default_value
# If no valid_types specified and variable has a value, return it # If no valid_types specified and variable has a value, return it
@@ -329,7 +401,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}' error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical: if critical:
raise TypeError(error_msg) raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}') log.error(f"[Ignored] {error_msg}")
return default_value return default_value
return variable return variable
scrapling/engines/toolbelt/fingerprints.py
+13 -13
View File
@@ -23,7 +23,7 @@ def generate_convincing_referer(url: str) -> str:
:return: Google's search URL of the domain name :return: Google's search URL of the domain name
""" """
website_name = extract(url).domain website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}' return f"https://www.google.com/search?q={website_name}"
@lru_cache(1, typed=True) @lru_cache(1, typed=True)
@@ -35,11 +35,11 @@ def get_os_name() -> Union[str, None]:
# #
os_name = platform.system() os_name = platform.system()
return { return {
'Linux': 'linux', "Linux": "linux",
'Darwin': 'macos', "Darwin": "macos",
'Windows': 'windows', "Windows": "windows",
# For the future? because why not # For the future? because why not
'iOS': 'ios', "iOS": "ios",
}.get(os_name) }.get(os_name)
@@ -50,9 +50,9 @@ def generate_suitable_fingerprint() -> Fingerprint:
:return: `Fingerprint` object :return: `Fingerprint` object
""" """
return FingerprintGenerator( return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)], browser=[Browser(name="chrome", min_version=128)],
os=get_os_name(), # None is ignored os=get_os_name(), # None is ignored
device='desktop' device="desktop",
).generate() ).generate()
@@ -67,15 +67,15 @@ def generate_headers(browser_mode: bool = False) -> Dict:
# So we don't raise any inconsistency red flags while websites fingerprinting us # So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name() os_name = get_os_name()
return HeaderGenerator( return HeaderGenerator(
browser=[Browser(name='chrome', min_version=130)], browser=[Browser(name="chrome", min_version=130)],
os=os_name, # None is ignored os=os_name, # None is ignored
device='desktop' device="desktop",
).generate() ).generate()
else: else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly # Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [ browsers = [
Browser(name='chrome', min_version=120), Browser(name="chrome", min_version=120),
Browser(name='firefox', min_version=120), Browser(name="firefox", min_version=120),
Browser(name='edge', min_version=120), Browser(name="edge", min_version=120),
] ]
return HeaderGenerator(browser=browsers, device='desktop').generate() return HeaderGenerator(browser=browsers, device="desktop").generate()
scrapling/engines/toolbelt/navigation.py
+29 -14
View File
@@ -1,6 +1,7 @@
""" """
Functions related to files and URLs Functions related to files and URLs
""" """
import os import os
from urllib.parse import urlencode, urlparse from urllib.parse import urlencode, urlparse
@@ -19,7 +20,9 @@ def intercept_route(route: Route):
:return: PlayWright `Route` object :return: PlayWright `Route` object
""" """
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
route.abort() route.abort()
else: else:
route.continue_() route.continue_()
@@ -32,7 +35,9 @@ async def async_intercept_route(route: async_Route):
:return: PlayWright `Route` object :return: PlayWright `Route` object
""" """
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES: if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"') log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
await route.abort() await route.abort()
else: else:
await route.continue_() await route.continue_()
@@ -50,23 +55,33 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict
proxy = urlparse(proxy_string) proxy = urlparse(proxy_string)
try: try:
return { return {
'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}', "server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",
'username': proxy.username or '', "username": proxy.username or "",
'password': proxy.password or '', "password": proxy.password or "",
} }
except ValueError: except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc... # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError('The proxy argument\'s string is in invalid format!') raise TypeError("The proxy argument's string is in invalid format!")
elif isinstance(proxy_string, dict): elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', ) valid_keys = (
if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()): "server",
"username",
"password",
)
if all(key in valid_keys for key in proxy_string.keys()) and not any(
key not in valid_keys for key in proxy_string.keys()
):
return proxy_string return proxy_string
else: else:
raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}') raise TypeError(
f"A proxy dictionary must have only these keys: {valid_keys}"
)
else: else:
raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!') raise TypeError(
f"Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!"
)
# The default value for proxy in Playwright's source is `None` # The default value for proxy in Playwright's source is `None`
return None return None
@@ -84,7 +99,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
parsed = urlparse(cdp_url) parsed = urlparse(cdp_url)
# Check scheme # Check scheme
if parsed.scheme not in ('ws', 'wss'): if parsed.scheme not in ("ws", "wss"):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port # Validate hostname and port
@@ -93,8 +108,8 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
# Ensure path starts with / # Ensure path starts with /
path = parsed.path path = parsed.path
if not path.startswith('/'): if not path.startswith("/"):
path = '/' + path path = "/" + path
# Reconstruct the base URL with validated parts # Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}" validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
@@ -118,4 +133,4 @@ def js_bypass_path(filename: str) -> str:
:return: The full path of the JS file. :return: The full path of the JS file.
""" """
current_directory = os.path.dirname(__file__) current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename) return os.path.join(current_directory, "bypasses", filename)
scrapling/fetchers.py
+329 -83
View File
@@ -1,7 +1,18 @@
from scrapling.core._types import (Callable, Dict, List, Literal, Optional, from scrapling.core._types import (
SelectorWaitStates, Union) Callable,
from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine, Dict,
check_if_engine_usable) List,
Literal,
Optional,
SelectorWaitStates,
Union,
)
from scrapling.engines import (
CamoufoxEngine,
PlaywrightEngine,
StaticEngine,
check_if_engine_usable,
)
from scrapling.engines.toolbelt import BaseFetcher, Response from scrapling.engines.toolbelt import BaseFetcher, Response
@@ -10,10 +21,19 @@ class Fetcher(BaseFetcher):
Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly. Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
""" """
@classmethod @classmethod
def get( def get(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP GET request for you but with some added flavors. """Make basic HTTP GET request for you but with some added flavors.
:param url: Target url. :param url: Target url.
@@ -30,16 +50,36 @@ class Fetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).get(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
adaptor_arguments=adaptor_arguments,
).get(**kwargs)
return response_object return response_object
@classmethod @classmethod
def post( def post(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP POST request for you but with some added flavors. """Make basic HTTP POST request for you but with some added flavors.
:param url: Target url. :param url: Target url.
@@ -56,16 +96,36 @@ class Fetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).post(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
adaptor_arguments=adaptor_arguments,
).post(**kwargs)
return response_object return response_object
@classmethod @classmethod
def put( def put(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors. """Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url :param url: Target url
@@ -83,16 +143,36 @@ class Fetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).put(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
adaptor_arguments=adaptor_arguments,
).put(**kwargs)
return response_object return response_object
@classmethod @classmethod
def delete( def delete(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors. """Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url :param url: Target url
@@ -109,18 +189,38 @@ class Fetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
adaptor_arguments=adaptor_arguments,
).delete(**kwargs)
return response_object return response_object
class AsyncFetcher(Fetcher): class AsyncFetcher(Fetcher):
@classmethod @classmethod
async def get( async def get(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP GET request for you but with some added flavors. """Make basic HTTP GET request for you but with some added flavors.
:param url: Target url. :param url: Target url.
@@ -137,16 +237,36 @@ class AsyncFetcher(Fetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_get(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
adaptor_arguments=adaptor_arguments,
).async_get(**kwargs)
return response_object return response_object
@classmethod @classmethod
async def post( async def post(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP POST request for you but with some added flavors. """Make basic HTTP POST request for you but with some added flavors.
:param url: Target url. :param url: Target url.
@@ -163,16 +283,36 @@ class AsyncFetcher(Fetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
adaptor_arguments=adaptor_arguments,
).async_post(**kwargs)
return response_object return response_object
@classmethod @classmethod
async def put( async def put(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors. """Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url :param url: Target url
@@ -189,16 +329,36 @@ class AsyncFetcher(Fetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_put(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
adaptor_arguments=adaptor_arguments,
).async_put(**kwargs)
return response_object return response_object
@classmethod @classmethod
async def delete( async def delete(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True, cls,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response: url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors. """Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url :param url: Target url
@@ -215,27 +375,57 @@ class AsyncFetcher(Fetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items()) adaptor_arguments = tuple(
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_delete(**kwargs) {**cls._generate_parser_arguments(), **custom_config}.items()
)
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
adaptor_arguments=adaptor_arguments,
).async_delete(**kwargs)
return response_object return response_object
class StealthyFetcher(BaseFetcher): class StealthyFetcher(BaseFetcher):
"""A `Fetcher` class type that is completely stealthy fetcher that uses a modified version of Firefox. """A `Fetcher` class type that is completely stealthy fetcher that uses a modified version of Firefox.
It works as real browsers passing almost all online tests/protections based on Camoufox. It works as real browsers passing almost all online tests/protections based on Camoufox.
Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain. Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
""" """
@classmethod @classmethod
def fetch( def fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, cls,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0, url: str,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, block_images: bool = False,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, disable_resources: bool = False,
custom_config: Dict = None, additional_arguments: Dict = None block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
addons: Optional[List[str]] = None,
wait: Optional[int] = 0,
timeout: Optional[float] = 30000,
page_action: Callable = None,
wait_selector: Optional[str] = None,
humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = "attached",
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
custom_config: Dict = None,
additional_arguments: Dict = None,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -271,7 +461,9 @@ class StealthyFetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = CamoufoxEngine( engine = CamoufoxEngine(
wait=wait, wait=wait,
@@ -294,18 +486,35 @@ class StealthyFetcher(BaseFetcher):
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {} additional_arguments=additional_arguments or {},
) )
return engine.fetch(url) return engine.fetch(url)
@classmethod @classmethod
async def async_fetch( async def async_fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, cls,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0, url: str,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, block_images: bool = False,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, disable_resources: bool = False,
custom_config: Dict = None, additional_arguments: Dict = None block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
addons: Optional[List[str]] = None,
wait: Optional[int] = 0,
timeout: Optional[float] = 30000,
page_action: Callable = None,
wait_selector: Optional[str] = None,
humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = "attached",
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
custom_config: Dict = None,
additional_arguments: Dict = None,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -341,7 +550,9 @@ class StealthyFetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = CamoufoxEngine( engine = CamoufoxEngine(
wait=wait, wait=wait,
@@ -364,7 +575,7 @@ class StealthyFetcher(BaseFetcher):
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {} additional_arguments=additional_arguments or {},
) )
return await engine.async_fetch(url) return await engine.async_fetch(url)
@@ -385,17 +596,32 @@ class PlayWrightFetcher(BaseFetcher):
> Note that these are the main options with PlayWright but it can be mixed together. > Note that these are the main options with PlayWright but it can be mixed together.
""" """
@classmethod @classmethod
def fetch( def fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, cls,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0, url: str,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', headless: Union[bool, str] = True,
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, disable_resources: bool = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', useragent: Optional[str] = None,
stealth: bool = False, real_chrome: bool = False, network_idle: bool = False,
cdp_url: Optional[str] = None, timeout: Optional[float] = 30000,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, wait: Optional[int] = 0,
custom_config: Dict = None page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
wait_selector_state: SelectorWaitStates = "attached",
hide_canvas: bool = False,
disable_webgl: bool = False,
extra_headers: Optional[Dict[str, str]] = None,
google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None,
locale: Optional[str] = "en-US",
stealth: bool = False,
real_chrome: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
custom_config: Dict = None,
) -> Response: ) -> Response:
"""Opens up a browser and do your request based on your chosen options below. """Opens up a browser and do your request based on your chosen options below.
@@ -428,7 +654,9 @@ class PlayWrightFetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = PlaywrightEngine( engine = PlaywrightEngine(
wait=wait, wait=wait,
@@ -457,15 +685,29 @@ class PlayWrightFetcher(BaseFetcher):
@classmethod @classmethod
async def async_fetch( async def async_fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, cls,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0, url: str,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', headless: Union[bool, str] = True,
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, disable_resources: bool = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', useragent: Optional[str] = None,
stealth: bool = False, real_chrome: bool = False, network_idle: bool = False,
cdp_url: Optional[str] = None, timeout: Optional[float] = 30000,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, wait: Optional[int] = 0,
custom_config: Dict = None page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
wait_selector_state: SelectorWaitStates = "attached",
hide_canvas: bool = False,
disable_webgl: bool = False,
extra_headers: Optional[Dict[str, str]] = None,
google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None,
locale: Optional[str] = "en-US",
stealth: bool = False,
real_chrome: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
custom_config: Dict = None,
) -> Response: ) -> Response:
"""Opens up a browser and do your request based on your chosen options below. """Opens up a browser and do your request based on your chosen options below.
@@ -498,7 +740,9 @@ class PlayWrightFetcher(BaseFetcher):
if not custom_config: if not custom_config:
custom_config = {} custom_config = {}
elif not isinstance(custom_config, dict): elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = PlaywrightEngine( engine = PlaywrightEngine(
wait=wait, wait=wait,
@@ -529,5 +773,7 @@ class PlayWrightFetcher(BaseFetcher):
class CustomFetcher(BaseFetcher): class CustomFetcher(BaseFetcher):
@classmethod @classmethod
def fetch(cls, url: str, browser_engine, **kwargs) -> Response: def fetch(cls, url: str, browser_engine, **kwargs) -> Response:
engine = check_if_engine_usable(browser_engine)(adaptor_arguments=cls._generate_parser_arguments(), **kwargs) engine = check_if_engine_usable(browser_engine)(
adaptor_arguments=cls._generate_parser_arguments(), **kwargs
)
return engine.fetch(url) return engine.fetch(url)
scrapling/parser.py
+452 -179
View File
File diff suppressed because it is too large Load Diff
setup.py
+10 -11
View File
@@ -1,7 +1,8 @@
from pathlib import Path
from setuptools import find_packages, setup from setuptools import find_packages, setup
with open("README.md", "r", encoding="utf-8") as fh: long_description = Path("README.md").read_text(encoding="utf-8")
long_description = fh.read()
setup( setup(
@@ -20,9 +21,7 @@ setup(
"scrapling": "scrapling", "scrapling": "scrapling",
}, },
entry_points={ entry_points={
'console_scripts': [ "console_scripts": ["scrapling=scrapling.cli:main"],
'scrapling=scrapling.cli:main'
],
}, },
include_package_data=True, include_package_data=True,
classifiers=[ classifiers=[
@@ -53,14 +52,14 @@ setup(
install_requires=[ install_requires=[
"lxml>=5.0", "lxml>=5.0",
"cssselect>=1.2", "cssselect>=1.2",
'click', "click",
"w3lib", "w3lib",
"orjson>=3", "orjson>=3",
"tldextract", "tldextract",
'httpx[brotli,zstd, socks]', "httpx[brotli,zstd, socks]",
'playwright>=1.49.1', "playwright>=1.49.1",
'rebrowser-playwright>=1.49.1', "rebrowser-playwright>=1.49.1",
'camoufox[geoip]>=0.4.11' "camoufox[geoip]>=0.4.11",
], ],
python_requires=">=3.9", python_requires=">=3.9",
url="https://github.com/D4Vinci/Scrapling", url="https://github.com/D4Vinci/Scrapling",
@@ -68,5 +67,5 @@ setup(
"Documentation": "https://scrapling.readthedocs.io/en/latest/", "Documentation": "https://scrapling.readthedocs.io/en/latest/",
"Source": "https://github.com/D4Vinci/Scrapling", "Source": "https://github.com/D4Vinci/Scrapling",
"Tracker": "https://github.com/D4Vinci/Scrapling/issues", "Tracker": "https://github.com/D4Vinci/Scrapling/issues",
} },
) )
tests/fetchers/async/test_camoufox.py
+55 -43
View File
@@ -17,43 +17,51 @@ class TestStealthyFetcher:
def urls(self, httpbin): def urls(self, httpbin):
url = httpbin.url url = httpbin.url
return { return {
'status_200': f'{url}/status/200', "status_200": f"{url}/status/200",
'status_404': f'{url}/status/404', "status_404": f"{url}/status/404",
'status_501': f'{url}/status/501', "status_501": f"{url}/status/501",
'basic_url': f'{url}/get', "basic_url": f"{url}/get",
'html_url': f'{url}/html', "html_url": f"{url}/html",
'delayed_url': f'{url}/delay/10', # 10 Seconds delay response "delayed_url": f"{url}/delay/10", # 10 Seconds delay response
'cookies_url': f"{url}/cookies/set/test/value" "cookies_url": f"{url}/cookies/set/test/value",
} }
async def test_basic_fetch(self, fetcher, urls): async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses""" """Test doing basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls['status_200'])).status == 200 assert (await fetcher.async_fetch(urls["status_200"])).status == 200
assert (await fetcher.async_fetch(urls['status_404'])).status == 404 assert (await fetcher.async_fetch(urls["status_404"])).status == 404
assert (await fetcher.async_fetch(urls['status_501'])).status == 501 assert (await fetcher.async_fetch(urls["status_501"])).status == 501
async def test_networkidle(self, fetcher, urls): async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not""" """Test if waiting for `networkidle` make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], network_idle=True)).status == 200 assert (
await fetcher.async_fetch(urls["basic_url"], network_idle=True)
).status == 200
async def test_blocking_resources(self, fetcher, urls): async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not""" """Test if blocking resources make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], block_images=True)).status == 200 assert (
assert (await fetcher.async_fetch(urls['basic_url'], disable_resources=True)).status == 200 await fetcher.async_fetch(urls["basic_url"], block_images=True)
).status == 200
assert (
await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
).status == 200
async def test_waiting_selector(self, fetcher, urls): async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not""" """Test if waiting for a selector make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['html_url'], wait_selector='h1')).status == 200 assert (
assert (await fetcher.async_fetch( await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
urls['html_url'], ).status == 200
wait_selector='h1', assert (
wait_selector_state='visible' await fetcher.async_fetch(
)).status == 200 urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
).status == 200
async def test_cookies_loading(self, fetcher, urls): async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url']) response = await fetcher.async_fetch(urls["cookies_url"])
assert response.cookies == {'test': 'value'} assert response.cookies == {"test": "value"}
async def test_automation(self, fetcher, urls): async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not""" """Test if automation break the code or not"""
@@ -64,34 +72,38 @@ class TestStealthyFetcher:
await page.mouse.up() await page.mouse.up()
return page return page
assert (await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)).status == 200 assert (
await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
).status == 200
async def test_properties(self, fetcher, urls): async def test_properties(self, fetcher, urls):
"""Test if different arguments breaks the code or not""" """Test if different arguments breaks the code or not"""
assert (await fetcher.async_fetch( assert (
urls['html_url'], await fetcher.async_fetch(
block_webrtc=True, urls["html_url"], block_webrtc=True, allow_webgl=True
allow_webgl=True )
)).status == 200 ).status == 200
assert (await fetcher.async_fetch( assert (
urls['html_url'], await fetcher.async_fetch(
block_webrtc=False, urls["html_url"], block_webrtc=False, allow_webgl=True
allow_webgl=True )
)).status == 200 ).status == 200
assert (await fetcher.async_fetch( assert (
urls['html_url'], await fetcher.async_fetch(
block_webrtc=True, urls["html_url"], block_webrtc=True, allow_webgl=False
allow_webgl=False )
)).status == 200 ).status == 200
assert (await fetcher.async_fetch( assert (
urls['html_url'], await fetcher.async_fetch(
extra_headers={'ayo': ''}, urls["html_url"], extra_headers={"ayo": ""}, os_randomize=True
os_randomize=True )
)).status == 200 ).status == 200
async def test_infinite_timeout(self, fetcher, urls): async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
assert (await fetcher.async_fetch(urls['delayed_url'], timeout=None)).status == 200 assert (
await fetcher.async_fetch(urls["delayed_url"], timeout=None)
).status == 200
tests/fetchers/async/test_httpx.py
+92 -51
View File
@@ -16,70 +16,111 @@ class TestAsyncFetcher:
@pytest.fixture(scope="class") @pytest.fixture(scope="class")
def urls(self, httpbin): def urls(self, httpbin):
return { return {
'status_200': f'{httpbin.url}/status/200', "status_200": f"{httpbin.url}/status/200",
'status_404': f'{httpbin.url}/status/404', "status_404": f"{httpbin.url}/status/404",
'status_501': f'{httpbin.url}/status/501', "status_501": f"{httpbin.url}/status/501",
'basic_url': f'{httpbin.url}/get', "basic_url": f"{httpbin.url}/get",
'post_url': f'{httpbin.url}/post', "post_url": f"{httpbin.url}/post",
'put_url': f'{httpbin.url}/put', "put_url": f"{httpbin.url}/put",
'delete_url': f'{httpbin.url}/delete', "delete_url": f"{httpbin.url}/delete",
'html_url': f'{httpbin.url}/html' "html_url": f"{httpbin.url}/html",
} }
async def test_basic_get(self, fetcher, urls): async def test_basic_get(self, fetcher, urls):
"""Test doing basic get request with multiple statuses""" """Test doing basic get request with multiple statuses"""
assert (await fetcher.get(urls['status_200'])).status == 200 assert (await fetcher.get(urls["status_200"])).status == 200
assert (await fetcher.get(urls['status_404'])).status == 404 assert (await fetcher.get(urls["status_404"])).status == 404
assert (await fetcher.get(urls['status_501'])).status == 501 assert (await fetcher.get(urls["status_501"])).status == 501
async def test_get_properties(self, fetcher, urls): async def test_get_properties(self, fetcher, urls):
"""Test if different arguments with GET request breaks the code or not""" """Test if different arguments with GET request breaks the code or not"""
assert (await fetcher.get(urls['status_200'], stealthy_headers=True)).status == 200 assert (
assert (await fetcher.get(urls['status_200'], follow_redirects=True)).status == 200 await fetcher.get(urls["status_200"], stealthy_headers=True)
assert (await fetcher.get(urls['status_200'], timeout=None)).status == 200 ).status == 200
assert (await fetcher.get( assert (
urls['status_200'], await fetcher.get(urls["status_200"], follow_redirects=True)
stealthy_headers=True, ).status == 200
follow_redirects=True, assert (await fetcher.get(urls["status_200"], timeout=None)).status == 200
timeout=None assert (
)).status == 200 await fetcher.get(
urls["status_200"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_post_properties(self, fetcher, urls): async def test_post_properties(self, fetcher, urls):
"""Test if different arguments with POST request breaks the code or not""" """Test if different arguments with POST request breaks the code or not"""
assert (await fetcher.post(urls['post_url'], data={'key': 'value'})).status == 200 assert (
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, stealthy_headers=True)).status == 200 await fetcher.post(urls["post_url"], data={"key": "value"})
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, follow_redirects=True)).status == 200 ).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, timeout=None)).status == 200 assert (
assert (await fetcher.post( await fetcher.post(
urls['post_url'], urls["post_url"], data={"key": "value"}, stealthy_headers=True
data={'key': 'value'}, )
stealthy_headers=True, ).status == 200
follow_redirects=True, assert (
timeout=None await fetcher.post(
)).status == 200 urls["post_url"], data={"key": "value"}, follow_redirects=True
)
).status == 200
assert (
await fetcher.post(urls["post_url"], data={"key": "value"}, timeout=None)
).status == 200
assert (
await fetcher.post(
urls["post_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_put_properties(self, fetcher, urls): async def test_put_properties(self, fetcher, urls):
"""Test if different arguments with PUT request breaks the code or not""" """Test if different arguments with PUT request breaks the code or not"""
assert (await fetcher.put(urls['put_url'], data={'key': 'value'})).status in [200, 405] assert (await fetcher.put(urls["put_url"], data={"key": "value"})).status in [
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, stealthy_headers=True)).status in [200, 405] 200,
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, follow_redirects=True)).status in [200, 405] 405,
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, timeout=None)).status in [200, 405] ]
assert (await fetcher.put( assert (
urls['put_url'], await fetcher.put(
data={'key': 'value'}, urls["put_url"], data={"key": "value"}, stealthy_headers=True
stealthy_headers=True, )
follow_redirects=True, ).status in [200, 405]
timeout=None assert (
)).status in [200, 405] await fetcher.put(
urls["put_url"], data={"key": "value"}, follow_redirects=True
)
).status in [200, 405]
assert (
await fetcher.put(urls["put_url"], data={"key": "value"}, timeout=None)
).status in [200, 405]
assert (
await fetcher.put(
urls["put_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status in [200, 405]
async def test_delete_properties(self, fetcher, urls): async def test_delete_properties(self, fetcher, urls):
"""Test if different arguments with DELETE request breaks the code or not""" """Test if different arguments with DELETE request breaks the code or not"""
assert (await fetcher.delete(urls['delete_url'], stealthy_headers=True)).status == 200 assert (
assert (await fetcher.delete(urls['delete_url'], follow_redirects=True)).status == 200 await fetcher.delete(urls["delete_url"], stealthy_headers=True)
assert (await fetcher.delete(urls['delete_url'], timeout=None)).status == 200 ).status == 200
assert (await fetcher.delete( assert (
urls['delete_url'], await fetcher.delete(urls["delete_url"], follow_redirects=True)
stealthy_headers=True, ).status == 200
follow_redirects=True, assert (await fetcher.delete(urls["delete_url"], timeout=None)).status == 200
timeout=None assert (
)).status == 200 await fetcher.delete(
urls["delete_url"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
tests/fetchers/async/test_playwright.py
+37 -27
View File
@@ -15,87 +15,97 @@ class TestPlayWrightFetcherAsync:
@pytest.fixture @pytest.fixture
def urls(self, httpbin): def urls(self, httpbin):
return { return {
'status_200': f'{httpbin.url}/status/200', "status_200": f"{httpbin.url}/status/200",
'status_404': f'{httpbin.url}/status/404', "status_404": f"{httpbin.url}/status/404",
'status_501': f'{httpbin.url}/status/501', "status_501": f"{httpbin.url}/status/501",
'basic_url': f'{httpbin.url}/get', "basic_url": f"{httpbin.url}/get",
'html_url': f'{httpbin.url}/html', "html_url": f"{httpbin.url}/html",
'delayed_url': f'{httpbin.url}/delay/10', "delayed_url": f"{httpbin.url}/delay/10",
'cookies_url': f"{httpbin.url}/cookies/set/test/value" "cookies_url": f"{httpbin.url}/cookies/set/test/value",
} }
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_basic_fetch(self, fetcher, urls): async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses""" """Test doing basic fetch request with multiple statuses"""
response = await fetcher.async_fetch(urls['status_200']) response = await fetcher.async_fetch(urls["status_200"])
assert response.status == 200 assert response.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_networkidle(self, fetcher, urls): async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not""" """Test if waiting for `networkidle` make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], network_idle=True) response = await fetcher.async_fetch(urls["basic_url"], network_idle=True)
assert response.status == 200 assert response.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_blocking_resources(self, fetcher, urls): async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not""" """Test if blocking resources make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], disable_resources=True) response = await fetcher.async_fetch(urls["basic_url"], disable_resources=True)
assert response.status == 200 assert response.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_waiting_selector(self, fetcher, urls): async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not""" """Test if waiting for a selector make page does not finish loading or not"""
response1 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1') response1 = await fetcher.async_fetch(urls["html_url"], wait_selector="h1")
assert response1.status == 200 assert response1.status == 200
response2 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1', wait_selector_state='visible') response2 = await fetcher.async_fetch(
urls["html_url"], wait_selector="h1", wait_selector_state="visible"
)
assert response2.status == 200 assert response2.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cookies_loading(self, fetcher, urls): async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url']) response = await fetcher.async_fetch(urls["cookies_url"])
assert response.cookies == {'test': 'value'} assert response.cookies == {"test": "value"}
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_automation(self, fetcher, urls): async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not""" """Test if automation break the code or not"""
async def scroll_page(page): async def scroll_page(page):
await page.mouse.wheel(10, 0) await page.mouse.wheel(10, 0)
await page.mouse.move(100, 400) await page.mouse.move(100, 400)
await page.mouse.up() await page.mouse.up()
return page return page
response = await fetcher.async_fetch(urls['html_url'], page_action=scroll_page) response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
assert response.status == 200 assert response.status == 200
@pytest.mark.parametrize("kwargs", [ @pytest.mark.parametrize(
{"disable_webgl": True, "hide_canvas": False}, "kwargs",
{"disable_webgl": False, "hide_canvas": True}, [
# {"stealth": True}, # causes issues with Github Actions {"disable_webgl": True, "hide_canvas": False},
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'}, {"disable_webgl": False, "hide_canvas": True},
{"extra_headers": {'ayo': ''}} # {"stealth": True}, # causes issues with Github Actions
]) {
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
},
{"extra_headers": {"ayo": ""}},
],
)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_properties(self, fetcher, urls, kwargs): async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments breaks the code or not""" """Test if different arguments breaks the code or not"""
response = await fetcher.async_fetch(urls['html_url'], **kwargs) response = await fetcher.async_fetch(urls["html_url"], **kwargs)
assert response.status == 200 assert response.status == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cdp_url_invalid(self, fetcher, urls): async def test_cdp_url_invalid(self, fetcher, urls):
"""Test if invalid CDP URLs raise appropriate exceptions""" """Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError): with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah') await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
with pytest.raises(ValueError): with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah', nstbrowser_mode=True) await fetcher.async_fetch(
urls["html_url"], cdp_url="blahblah", nstbrowser_mode=True
)
with pytest.raises(Exception): with pytest.raises(Exception):
await fetcher.async_fetch(urls['html_url'], cdp_url='ws://blahblah') await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_infinite_timeout(self, fetcher, urls): async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
response = await fetcher.async_fetch(urls['delayed_url'], timeout=None) response = await fetcher.async_fetch(urls["delayed_url"], timeout=None)
assert response.status == 200 assert response.status == 200
tests/fetchers/sync/test_camoufox.py
+33 -13
View File
@@ -16,12 +16,12 @@ class TestStealthyFetcher:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def setup_urls(self, httpbin): def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing""" """Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200' self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f'{httpbin.url}/status/404' self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f'{httpbin.url}/status/501' self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f'{httpbin.url}/get' self.basic_url = f"{httpbin.url}/get"
self.html_url = f'{httpbin.url}/html' self.html_url = f"{httpbin.url}/html"
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value" self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher): def test_basic_fetch(self, fetcher):
@@ -41,15 +41,21 @@ class TestStealthyFetcher:
def test_waiting_selector(self, fetcher): def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not""" """Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200 assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200 assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher): def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'} assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
def test_automation(self, fetcher): def test_automation(self, fetcher):
"""Test if automation break the code or not""" """Test if automation break the code or not"""
def scroll_page(page): def scroll_page(page):
page.mouse.wheel(10, 0) page.mouse.wheel(10, 0)
page.mouse.move(100, 400) page.mouse.move(100, 400)
@@ -60,10 +66,24 @@ class TestStealthyFetcher:
def test_properties(self, fetcher): def test_properties(self, fetcher):
"""Test if different arguments breaks the code or not""" """Test if different arguments breaks the code or not"""
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status == 200 assert (
assert fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status == 200 fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status == 200 == 200
assert fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status == 200 )
assert (
fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status
== 200
)
assert (
fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status
== 200
)
assert (
fetcher.fetch(
self.html_url, extra_headers={"ayo": ""}, os_randomize=True
).status
== 200
)
def test_infinite_timeout(self, fetcher): def test_infinite_timeout(self, fetcher):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
tests/fetchers/sync/test_httpx.py
+79 -42
View File
@@ -16,14 +16,14 @@ class TestFetcher:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def setup_urls(self, httpbin): def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing""" """Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200' self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f'{httpbin.url}/status/404' self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f'{httpbin.url}/status/501' self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f'{httpbin.url}/get' self.basic_url = f"{httpbin.url}/get"
self.post_url = f'{httpbin.url}/post' self.post_url = f"{httpbin.url}/post"
self.put_url = f'{httpbin.url}/put' self.put_url = f"{httpbin.url}/put"
self.delete_url = f'{httpbin.url}/delete' self.delete_url = f"{httpbin.url}/delete"
self.html_url = f'{httpbin.url}/html' self.html_url = f"{httpbin.url}/html"
def test_basic_get(self, fetcher): def test_basic_get(self, fetcher):
"""Test doing basic get request with multiple statuses""" """Test doing basic get request with multiple statuses"""
@@ -36,49 +36,86 @@ class TestFetcher:
assert fetcher.get(self.status_200, stealthy_headers=True).status == 200 assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
assert fetcher.get(self.status_200, follow_redirects=True).status == 200 assert fetcher.get(self.status_200, follow_redirects=True).status == 200
assert fetcher.get(self.status_200, timeout=None).status == 200 assert fetcher.get(self.status_200, timeout=None).status == 200
assert fetcher.get( assert (
self.status_200, fetcher.get(
stealthy_headers=True, self.status_200,
follow_redirects=True, stealthy_headers=True,
timeout=None follow_redirects=True,
).status == 200 timeout=None,
).status
== 200
)
def test_post_properties(self, fetcher): def test_post_properties(self, fetcher):
"""Test if different arguments with POST request breaks the code or not""" """Test if different arguments with POST request breaks the code or not"""
assert fetcher.post(self.post_url, data={'key': 'value'}).status == 200 assert fetcher.post(self.post_url, data={"key": "value"}).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status == 200 assert (
assert fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status == 200 fetcher.post(
assert fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status == 200 self.post_url, data={"key": "value"}, stealthy_headers=True
assert fetcher.post( ).status
self.post_url, == 200
data={'key': 'value'}, )
stealthy_headers=True, assert (
follow_redirects=True, fetcher.post(
timeout=None self.post_url, data={"key": "value"}, follow_redirects=True
).status == 200 ).status
== 200
)
assert (
fetcher.post(self.post_url, data={"key": "value"}, timeout=None).status
== 200
)
assert (
fetcher.post(
self.post_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_put_properties(self, fetcher): def test_put_properties(self, fetcher):
"""Test if different arguments with PUT request breaks the code or not""" """Test if different arguments with PUT request breaks the code or not"""
assert fetcher.put(self.put_url, data={'key': 'value'}).status == 200 assert fetcher.put(self.put_url, data={"key": "value"}).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status == 200 assert (
assert fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status == 200 fetcher.put(
assert fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status == 200 self.put_url, data={"key": "value"}, stealthy_headers=True
assert fetcher.put( ).status
self.put_url, == 200
data={'key': 'value'}, )
stealthy_headers=True, assert (
follow_redirects=True, fetcher.put(
timeout=None self.put_url, data={"key": "value"}, follow_redirects=True
).status == 200 ).status
== 200
)
assert (
fetcher.put(self.put_url, data={"key": "value"}, timeout=None).status == 200
)
assert (
fetcher.put(
self.put_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_delete_properties(self, fetcher): def test_delete_properties(self, fetcher):
"""Test if different arguments with DELETE request breaks the code or not""" """Test if different arguments with DELETE request breaks the code or not"""
assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200 assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200 assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
assert fetcher.delete(self.delete_url, timeout=None).status == 200 assert fetcher.delete(self.delete_url, timeout=None).status == 200
assert fetcher.delete( assert (
self.delete_url, fetcher.delete(
stealthy_headers=True, self.delete_url,
follow_redirects=True, stealthy_headers=True,
timeout=None follow_redirects=True,
).status == 200 timeout=None,
).status
== 200
)
tests/fetchers/sync/test_playwright.py
+33 -21
View File
@@ -8,7 +8,6 @@ PlayWrightFetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
class TestPlayWrightFetcher: class TestPlayWrightFetcher:
@pytest.fixture(scope="class") @pytest.fixture(scope="class")
def fetcher(self): def fetcher(self):
"""Fixture to create a StealthyFetcher instance for the entire test class""" """Fixture to create a StealthyFetcher instance for the entire test class"""
@@ -17,12 +16,12 @@ class TestPlayWrightFetcher:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def setup_urls(self, httpbin): def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing""" """Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200' self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f'{httpbin.url}/status/404' self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f'{httpbin.url}/status/501' self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f'{httpbin.url}/get' self.basic_url = f"{httpbin.url}/get"
self.html_url = f'{httpbin.url}/html' self.html_url = f"{httpbin.url}/html"
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value" self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher): def test_basic_fetch(self, fetcher):
@@ -42,12 +41,17 @@ class TestPlayWrightFetcher:
def test_waiting_selector(self, fetcher): def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not""" """Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200 assert fetcher.fetch(self.html_url, wait_selector="h1").status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200 assert (
fetcher.fetch(
self.html_url, wait_selector="h1", wait_selector_state="visible"
).status
== 200
)
def test_cookies_loading(self, fetcher): def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request""" """Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'} assert fetcher.fetch(self.cookies_url).cookies == {"test": "value"}
def test_automation(self, fetcher): def test_automation(self, fetcher):
"""Test if automation break the code or not""" """Test if automation break the code or not"""
@@ -60,13 +64,18 @@ class TestPlayWrightFetcher:
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200 assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
@pytest.mark.parametrize("kwargs", [ @pytest.mark.parametrize(
{"disable_webgl": True, "hide_canvas": False}, "kwargs",
{"disable_webgl": False, "hide_canvas": True}, [
# {"stealth": True}, # causes issues with Github Actions {"disable_webgl": True, "hide_canvas": False},
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'}, {"disable_webgl": False, "hide_canvas": True},
{"extra_headers": {'ayo': ''}} # {"stealth": True}, # causes issues with Github Actions
]) {
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"
},
{"extra_headers": {"ayo": ""}},
],
)
def test_properties(self, fetcher, kwargs): def test_properties(self, fetcher, kwargs):
"""Test if different arguments breaks the code or not""" """Test if different arguments breaks the code or not"""
response = fetcher.fetch(self.html_url, **kwargs) response = fetcher.fetch(self.html_url, **kwargs)
@@ -75,15 +84,18 @@ class TestPlayWrightFetcher:
def test_cdp_url_invalid(self, fetcher): def test_cdp_url_invalid(self, fetcher):
"""Test if invalid CDP URLs raise appropriate exceptions""" """Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError): with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah') fetcher.fetch(self.html_url, cdp_url="blahblah")
with pytest.raises(ValueError): with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True) fetcher.fetch(self.html_url, cdp_url="blahblah", nstbrowser_mode=True)
with pytest.raises(Exception): with pytest.raises(Exception):
fetcher.fetch(self.html_url, cdp_url='ws://blahblah') fetcher.fetch(self.html_url, cdp_url="ws://blahblah")
def test_infinite_timeout(self, fetcher, ): def test_infinite_timeout(
self,
fetcher,
):
"""Test if infinite timeout breaks the code or not""" """Test if infinite timeout breaks the code or not"""
response = fetcher.fetch(self.delayed_url, timeout=None) response = fetcher.fetch(self.delayed_url, timeout=None)
assert response.status == 200 assert response.status == 200
tests/fetchers/test_utils.py
+105 -64
View File
@@ -7,76 +7,117 @@ from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
def content_type_map(): def content_type_map():
return { return {
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
'text/html; charset=UTF-8': 'UTF-8', "text/html; charset=UTF-8": "UTF-8",
'text/html; charset=ISO-8859-1': 'ISO-8859-1', "text/html; charset=ISO-8859-1": "ISO-8859-1",
'text/html': 'ISO-8859-1', "text/html": "ISO-8859-1",
'application/json; charset=UTF-8': 'UTF-8', "application/json; charset=UTF-8": "UTF-8",
'application/json': 'utf-8', "application/json": "utf-8",
'text/json': 'utf-8', "text/json": "utf-8",
'application/javascript; charset=UTF-8': 'UTF-8', "application/javascript; charset=UTF-8": "UTF-8",
'application/javascript': 'utf-8', "application/javascript": "utf-8",
'text/plain; charset=UTF-8': 'UTF-8', "text/plain; charset=UTF-8": "UTF-8",
'text/plain; charset=ISO-8859-1': 'ISO-8859-1', "text/plain; charset=ISO-8859-1": "ISO-8859-1",
'text/plain': 'ISO-8859-1', "text/plain": "ISO-8859-1",
'application/xhtml+xml; charset=UTF-8': 'UTF-8', "application/xhtml+xml; charset=UTF-8": "UTF-8",
'application/xhtml+xml': 'utf-8', "application/xhtml+xml": "utf-8",
'text/html; charset=windows-1252': 'windows-1252', "text/html; charset=windows-1252": "windows-1252",
'application/json; charset=windows-1252': 'windows-1252', "application/json; charset=windows-1252": "windows-1252",
'text/plain; charset=windows-1252': 'windows-1252', "text/plain; charset=windows-1252": "windows-1252",
'text/html; charset="UTF-8"': 'UTF-8', 'text/html; charset="UTF-8"': "UTF-8",
'text/html; charset="ISO-8859-1"': 'ISO-8859-1', 'text/html; charset="ISO-8859-1"': "ISO-8859-1",
'text/html; charset="windows-1252"': 'windows-1252', 'text/html; charset="windows-1252"': "windows-1252",
'application/json; charset="UTF-8"': 'UTF-8', 'application/json; charset="UTF-8"': "UTF-8",
'application/json; charset="ISO-8859-1"': 'ISO-8859-1', 'application/json; charset="ISO-8859-1"': "ISO-8859-1",
'application/json; charset="windows-1252"': 'windows-1252', 'application/json; charset="windows-1252"': "windows-1252",
'text/json; charset="UTF-8"': 'UTF-8', 'text/json; charset="UTF-8"': "UTF-8",
'application/javascript; charset="UTF-8"': 'UTF-8', 'application/javascript; charset="UTF-8"': "UTF-8",
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1', 'application/javascript; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="UTF-8"': 'UTF-8', 'text/plain; charset="UTF-8"': "UTF-8",
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1', 'text/plain; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="windows-1252"': 'windows-1252', 'text/plain; charset="windows-1252"': "windows-1252",
'application/xhtml+xml; charset="UTF-8"': 'UTF-8', 'application/xhtml+xml; charset="UTF-8"': "UTF-8",
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1', 'application/xhtml+xml; charset="ISO-8859-1"': "ISO-8859-1",
'application/xhtml+xml; charset="windows-1252"': 'windows-1252', 'application/xhtml+xml; charset="windows-1252"': "windows-1252",
'text/html; charset="US-ASCII"': 'US-ASCII', 'text/html; charset="US-ASCII"': "US-ASCII",
'application/json; charset="US-ASCII"': 'US-ASCII', 'application/json; charset="US-ASCII"': "US-ASCII",
'text/plain; charset="US-ASCII"': 'US-ASCII', 'text/plain; charset="US-ASCII"': "US-ASCII",
'text/html; charset="Shift_JIS"': 'Shift_JIS', 'text/html; charset="Shift_JIS"': "Shift_JIS",
'application/json; charset="Shift_JIS"': 'Shift_JIS', 'application/json; charset="Shift_JIS"': "Shift_JIS",
'text/plain; charset="Shift_JIS"': 'Shift_JIS', 'text/plain; charset="Shift_JIS"': "Shift_JIS",
'application/xml; charset="UTF-8"': 'UTF-8', 'application/xml; charset="UTF-8"': "UTF-8",
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1', 'application/xml; charset="ISO-8859-1"': "ISO-8859-1",
'application/xml': 'utf-8', "application/xml": "utf-8",
'text/xml; charset="UTF-8"': 'UTF-8', 'text/xml; charset="UTF-8"': "UTF-8",
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1', 'text/xml; charset="ISO-8859-1"': "ISO-8859-1",
'text/xml': 'utf-8' "text/xml": "utf-8",
} }
@pytest.fixture @pytest.fixture
def status_map(): def status_map():
return { return {
100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints", 100: "Continue",
200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information", 101: "Switching Protocols",
204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status", 102: "Processing",
208: "Already Reported", 226: "IM Used", 300: "Multiple Choices", 103: "Early Hints",
301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified", 200: "OK",
305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect", 201: "Created",
400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden", 202: "Accepted",
404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable", 203: "Non-Authoritative Information",
407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict", 204: "No Content",
410: "Gone", 411: "Length Required", 412: "Precondition Failed", 205: "Reset Content",
413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type", 206: "Partial Content",
416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot", 207: "Multi-Status",
421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked", 208: "Already Reported",
424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required", 226: "IM Used",
428: "Precondition Required", 429: "Too Many Requests", 300: "Multiple Choices",
431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons", 301: "Moved Permanently",
500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway", 302: "Found",
503: "Service Unavailable", 504: "Gateway Timeout", 303: "See Other",
505: "HTTP Version Not Supported", 506: "Variant Also Negotiates", 304: "Not Modified",
507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended", 305: "Use Proxy",
511: "Network Authentication Required" 307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
} }
tests/parser/test_automatch.py
+22 -22
View File
@@ -8,7 +8,7 @@ from scrapling import Adaptor
class TestParserAutoMatch: class TestParserAutoMatch:
def test_element_relocation(self): def test_element_relocation(self):
"""Test relocating element after structure change""" """Test relocating element after structure change"""
original_html = ''' original_html = """
<div class="container"> <div class="container">
<section class="products"> <section class="products">
<article class="product" id="p1"> <article class="product" id="p1">
@@ -21,8 +21,8 @@ class TestParserAutoMatch:
</article> </article>
</section> </section>
</div> </div>
''' """
changed_html = ''' changed_html = """
<div class="new-container"> <div class="new-container">
<div class="product-wrapper"> <div class="product-wrapper">
<section class="products"> <section class="products">
@@ -41,25 +41,25 @@ class TestParserAutoMatch:
</section> </section>
</div> </div>
</div> </div>
''' """
old_page = Adaptor(original_html, url='example.com', auto_match=True) old_page = Adaptor(original_html, url="example.com", auto_match=True)
new_page = Adaptor(changed_html, url='example.com', auto_match=True) new_page = Adaptor(changed_html, url="example.com", auto_match=True)
# 'p1' was used as ID and now it's not and all the path elements have changes # 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors # Also at the same time testing auto-match vs combined selectors
_ = old_page.css('#p1, #p2', auto_save=True)[0] _ = old_page.css("#p1, #p2", auto_save=True)[0]
relocated = new_page.css('#p1', auto_match=True) relocated = new_page.css("#p1", auto_match=True)
assert relocated is not None assert relocated is not None
assert relocated[0].attrib['data-id'] == 'p1' assert relocated[0].attrib["data-id"] == "p1"
assert relocated[0].has_class('new-class') assert relocated[0].has_class("new-class")
assert relocated[0].css('.new-description')[0].text == 'Description 1' assert relocated[0].css(".new-description")[0].text == "Description 1"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_element_relocation_async(self): async def test_element_relocation_async(self):
"""Test relocating element after structure change in async mode""" """Test relocating element after structure change in async mode"""
original_html = ''' original_html = """
<div class="container"> <div class="container">
<section class="products"> <section class="products">
<article class="product" id="p1"> <article class="product" id="p1">
@@ -72,8 +72,8 @@ class TestParserAutoMatch:
</article> </article>
</section> </section>
</div> </div>
''' """
changed_html = ''' changed_html = """
<div class="new-container"> <div class="new-container">
<div class="product-wrapper"> <div class="product-wrapper">
<section class="products"> <section class="products">
@@ -92,20 +92,20 @@ class TestParserAutoMatch:
</section> </section>
</div> </div>
</div> </div>
''' """
# Simulate async operation # Simulate async operation
await asyncio.sleep(0.1) # Minimal async operation await asyncio.sleep(0.1) # Minimal async operation
old_page = Adaptor(original_html, url='example.com', auto_match=True) old_page = Adaptor(original_html, url="example.com", auto_match=True)
new_page = Adaptor(changed_html, url='example.com', auto_match=True) new_page = Adaptor(changed_html, url="example.com", auto_match=True)
# 'p1' was used as ID and now it's not and all the path elements have changes # 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors # Also at the same time testing auto-match vs combined selectors
_ = old_page.css('#p1, #p2', auto_save=True)[0] _ = old_page.css("#p1, #p2", auto_save=True)[0]
relocated = new_page.css('#p1', auto_match=True) relocated = new_page.css("#p1", auto_match=True)
assert relocated is not None assert relocated is not None
assert relocated[0].attrib['data-id'] == 'p1' assert relocated[0].attrib["data-id"] == "p1"
assert relocated[0].has_class('new-class') assert relocated[0].has_class("new-class")
assert relocated[0].css('.new-description')[0].text == 'Description 1' assert relocated[0].css(".new-description")[0].text == "Description 1"
tests/parser/test_general.py
+61 -49
View File
@@ -9,7 +9,7 @@ from scrapling import Adaptor
@pytest.fixture @pytest.fixture
def html_content(): def html_content():
return ''' return """
<html> <html>
<head> <head>
<title>Complex Web Page</title> <title>Complex Web Page</title>
@@ -73,7 +73,7 @@ def html_content():
</script> </script>
</body> </body>
</html> </html>
''' """
@pytest.fixture @pytest.fixture
@@ -85,13 +85,14 @@ def page(html_content):
class TestCSSSelectors: class TestCSSSelectors:
def test_basic_product_selection(self, page): def test_basic_product_selection(self, page):
"""Test selecting all product elements""" """Test selecting all product elements"""
elements = page.css('main #products .product-list article.product') elements = page.css("main #products .product-list article.product")
assert len(elements) == 3 assert len(elements) == 3
def test_in_stock_product_selection(self, page): def test_in_stock_product_selection(self, page):
"""Test selecting in-stock products""" """Test selecting in-stock products"""
in_stock_products = page.css( in_stock_products = page.css(
'main #products .product-list article.product:not(:contains("Out of stock"))') 'main #products .product-list article.product:not(:contains("Out of stock"))'
)
assert len(in_stock_products) == 2 assert len(in_stock_products) == 2
@@ -117,22 +118,26 @@ class TestXPathSelectors:
class TestTextMatching: class TestTextMatching:
def test_regex_multiple_matches(self, page): def test_regex_multiple_matches(self, page):
"""Test finding multiple matches with regex""" """Test finding multiple matches with regex"""
stock_info = page.find_by_regex(r'In stock: \d+', first_match=False) stock_info = page.find_by_regex(r"In stock: \d+", first_match=False)
assert len(stock_info) == 2 assert len(stock_info) == 2
def test_regex_first_match(self, page): def test_regex_first_match(self, page):
"""Test finding the first match with regex""" """Test finding the first match with regex"""
stock_info = page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True) stock_info = page.find_by_regex(
assert stock_info.text == 'In stock: 5' r"In stock: \d+", first_match=True, case_sensitive=True
)
assert stock_info.text == "In stock: 5"
def test_partial_text_match(self, page): def test_partial_text_match(self, page):
"""Test finding elements with partial text match""" """Test finding elements with partial text match"""
stock_info = page.find_by_text(r'In stock:', partial=True, first_match=False) stock_info = page.find_by_text(r"In stock:", partial=True, first_match=False)
assert len(stock_info) == 2 assert len(stock_info) == 2
def test_exact_text_match(self, page): def test_exact_text_match(self, page):
"""Test finding elements with exact text match""" """Test finding elements with exact text match"""
out_of_stock = page.find_by_text('Out of stock', partial=False, first_match=False) out_of_stock = page.find_by_text(
"Out of stock", partial=False, first_match=False
)
assert len(out_of_stock) == 1 assert len(out_of_stock) == 1
@@ -140,17 +145,17 @@ class TestTextMatching:
class TestSimilarElements: class TestSimilarElements:
def test_finding_similar_products(self, page): def test_finding_similar_products(self, page):
"""Test finding similar product elements""" """Test finding similar product elements"""
first_product = page.css_first('.product') first_product = page.css_first(".product")
similar_products = first_product.find_similar() similar_products = first_product.find_similar()
assert len(similar_products) == 2 assert len(similar_products) == 2
def test_finding_similar_reviews(self, page): def test_finding_similar_reviews(self, page):
"""Test finding similar review elements with additional filtering""" """Test finding similar review elements with additional filtering"""
first_review = page.find('div', class_='review') first_review = page.find("div", class_="review")
similar_high_rated_reviews = [ similar_high_rated_reviews = [
review review
for review in first_review.find_similar() for review in first_review.find_similar()
if int(review.attrib.get('data-rating', 0)) >= 4 if int(review.attrib.get("data-rating", 0)) >= 4
] ]
assert len(similar_high_rated_reviews) == 1 assert len(similar_high_rated_reviews) == 1
@@ -181,17 +186,17 @@ class TestErrorHandling:
def test_bad_selectors(self, page): def test_bad_selectors(self, page):
"""Test handling of invalid selectors""" """Test handling of invalid selectors"""
with pytest.raises((SelectorError, SelectorSyntaxError)): with pytest.raises((SelectorError, SelectorSyntaxError)):
page.css('4 ayo') page.css("4 ayo")
with pytest.raises((SelectorError, SelectorSyntaxError)): with pytest.raises((SelectorError, SelectorSyntaxError)):
page.xpath('4 ayo') page.xpath("4 ayo")
# Pickling and Object Representation Tests # Pickling and Object Representation Tests
class TestPicklingAndRepresentation: class TestPicklingAndRepresentation:
def test_unpickleable_objects(self, page): def test_unpickleable_objects(self, page):
"""Test that Adaptor objects cannot be pickled""" """Test that Adaptor objects cannot be pickled"""
table = page.css('.product-list')[0] table = page.css(".product-list")[0]
with pytest.raises(TypeError): with pytest.raises(TypeError):
pickle.dumps(table) pickle.dumps(table)
@@ -200,7 +205,7 @@ class TestPicklingAndRepresentation:
def test_string_representations(self, page): def test_string_representations(self, page):
"""Test custom string representations of objects""" """Test custom string representations of objects"""
table = page.css('.product-list')[0] table = page.css(".product-list")[0]
assert issubclass(type(table.__str__()), str) assert issubclass(type(table.__str__()), str)
assert issubclass(type(table.__repr__()), str) assert issubclass(type(table.__repr__()), str)
assert issubclass(type(table.attrib.__str__()), str) assert issubclass(type(table.attrib.__str__()), str)
@@ -211,40 +216,40 @@ class TestPicklingAndRepresentation:
class TestElementNavigation: class TestElementNavigation:
def test_basic_navigation_properties(self, page): def test_basic_navigation_properties(self, page):
"""Test basic navigation properties of elements""" """Test basic navigation properties of elements"""
table = page.css('.product-list')[0] table = page.css(".product-list")[0]
assert table.path is not None assert table.path is not None
assert table.html_content != '' assert table.html_content != ""
assert table.prettify() != '' assert table.prettify() != ""
def test_parent_and_sibling_navigation(self, page): def test_parent_and_sibling_navigation(self, page):
"""Test parent and sibling navigation""" """Test parent and sibling navigation"""
table = page.css('.product-list')[0] table = page.css(".product-list")[0]
parent = table.parent parent = table.parent
assert parent.attrib['id'] == 'products' assert parent.attrib["id"] == "products"
parent_siblings = parent.siblings parent_siblings = parent.siblings
assert len(parent_siblings) == 1 assert len(parent_siblings) == 1
def test_child_navigation(self, page): def test_child_navigation(self, page):
"""Test child navigation""" """Test child navigation"""
table = page.css('.product-list')[0] table = page.css(".product-list")[0]
children = table.children children = table.children
assert len(children) == 3 assert len(children) == 3
def test_next_and_previous_navigation(self, page): def test_next_and_previous_navigation(self, page):
"""Test next and previous element navigation""" """Test next and previous element navigation"""
child = page.css('.product-list')[0].find({'data-id': "1"}) child = page.css(".product-list")[0].find({"data-id": "1"})
next_element = child.next next_element = child.next
assert next_element.attrib['data-id'] == '2' assert next_element.attrib["data-id"] == "2"
prev_element = next_element.previous prev_element = next_element.previous
assert prev_element.tag == child.tag assert prev_element.tag == child.tag
def test_ancestor_finding(self, page): def test_ancestor_finding(self, page):
"""Test finding ancestors of elements""" """Test finding ancestors of elements"""
all_prices = page.css('.price') all_prices = page.css(".price")
products_with_prices = [ products_with_prices = [
price.find_ancestor(lambda p: p.has_class('product')) price.find_ancestor(lambda p: p.has_class("product"))
for price in all_prices for price in all_prices
] ]
assert len(products_with_prices) == 3 assert len(products_with_prices) == 3
@@ -254,52 +259,59 @@ class TestElementNavigation:
class TestJSONAndAttributes: class TestJSONAndAttributes:
def test_json_conversion(self, page): def test_json_conversion(self, page):
"""Test converting content to JSON""" """Test converting content to JSON"""
script_content = page.css('#page-data::text')[0] script_content = page.css("#page-data::text")[0]
assert issubclass(type(script_content.sort()), str) assert issubclass(type(script_content.sort()), str)
page_data = script_content.json() page_data = script_content.json()
assert page_data['totalProducts'] == 3 assert page_data["totalProducts"] == 3
assert 'lastUpdated' in page_data assert "lastUpdated" in page_data
def test_attribute_operations(self, page): def test_attribute_operations(self, page):
"""Test various attribute-related operations""" """Test various attribute-related operations"""
# Product ID extraction # Product ID extraction
products = page.css('.product') products = page.css(".product")
product_ids = [product.attrib['data-id'] for product in products] product_ids = [product.attrib["data-id"] for product in products]
assert product_ids == ['1', '2', '3'] assert product_ids == ["1", "2", "3"]
assert 'data-id' in products[0].attrib assert "data-id" in products[0].attrib
# Review rating calculations # Review rating calculations
reviews = page.css('.review') reviews = page.css(".review")
review_ratings = [int(review.attrib['data-rating']) for review in reviews] review_ratings = [int(review.attrib["data-rating"]) for review in reviews]
assert sum(review_ratings) / len(review_ratings) == 4.5 assert sum(review_ratings) / len(review_ratings) == 4.5
# Attribute searching # Attribute searching
key_value = list(products[0].attrib.search_values('1', partial=False)) key_value = list(products[0].attrib.search_values("1", partial=False))
assert list(key_value[0].keys()) == ['data-id'] assert list(key_value[0].keys()) == ["data-id"]
key_value = list(products[0].attrib.search_values('1', partial=True)) key_value = list(products[0].attrib.search_values("1", partial=True))
assert list(key_value[0].keys()) == ['data-id'] assert list(key_value[0].keys()) == ["data-id"]
# JSON attribute conversion # JSON attribute conversion
attr_json = page.css_first('#products').attrib['schema'].json() attr_json = page.css_first("#products").attrib["schema"].json()
assert attr_json == {'jsonable': 'data'} assert attr_json == {"jsonable": "data"}
assert isinstance(page.css('#products')[0].attrib.json_string, bytes) assert isinstance(page.css("#products")[0].attrib.json_string, bytes)
# Performance Test # Performance Test
def test_large_html_parsing_performance(): def test_large_html_parsing_performance():
"""Test parsing and selecting performance on large HTML""" """Test parsing and selecting performance on large HTML"""
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>' large_html = (
"<html><body>"
+ '<div class="item">' * 5000
+ "</div>" * 5000
+ "</body></html>"
)
start_time = time.time() start_time = time.time()
parsed = Adaptor(large_html, auto_match=False) parsed = Adaptor(large_html, auto_match=False)
elements = parsed.css('.item') elements = parsed.css(".item")
end_time = time.time() end_time = time.time()
assert len(elements) == 5000 assert len(elements) == 5000
# Converting 5000 elements to a class and doing operations on them will take time # Converting 5000 elements to a class and doing operations on them will take time
# Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
assert end_time - start_time < 0.5 # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds assert (
end_time - start_time < 0.5
) # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
# Selector Generation Test # Selector Generation Test
@@ -318,13 +330,13 @@ def test_selectors_generation(page):
# Miscellaneous Tests # Miscellaneous Tests
def test_getting_all_text(page): def test_getting_all_text(page):
"""Test getting all text from the page""" """Test getting all text from the page"""
assert page.get_all_text() != '' assert page.get_all_text() != ""
def test_regex_on_text(page): def test_regex_on_text(page):
"""Test regex operations on text""" """Test regex operations on text"""
element = page.css('[data-id="1"] .price')[0] element = page.css('[data-id="1"] .price')[0]
match = element.re_first(r'[\.\d]+') match = element.re_first(r"[\.\d]+")
assert match == '10.99' assert match == "10.99"
match = element.text.re(r'(\d+)', replace_entities=False) match = element.text.re(r"(\d+)", replace_entities=False)
assert len(match) == 2 assert len(match) == 2