freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: huge change, many features/class got a better naming

Browse Source 
- `Adaptor` became `Selector`
- `Adaptors` became `Selectors`
- `auto_match` argument/feature became `adaptive`
- `adaptor_arguments` argument became `selector_config`
- `automatch_domain` argument became `adaptive_domain`
- `additional_arguments` argument became `additional_args`
- `storage_adaptors` file became just `storage`
This commit is contained in:
Karim shoair
2025-07-29 04:20:23 +03:00
parent 0c649987f8
commit 264ae02aa7
22 changed files with 250 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -5
View File
@@ -52,14 +52,14 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
```python ```python
>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher >> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
>> StealthyFetcher.auto_match = True >> StealthyFetcher.adaptive = True
# Fetch websites' source under the radar! # Fetch websites' source under the radar!
>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) >> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)
>> print(page.status) >> print(page.status)
200 200
>> products = page.css('.product', auto_save=True) # Scrape data that survives website design changes! >> products = page.css('.product', auto_save=True) # Scrape data that survives website design changes!
>> # Later, if the website structure changes, pass `auto_match=True` >> # Later, if the website structure changes, pass `adaptive=True`
>> products = page.css('.product', auto_match=True) # and Scrapling still finds them! >> products = page.css('.product', adaptive=True) # and Scrapling still finds them!
``` ```
# Sponsors # Sponsors
@@ -150,7 +150,7 @@ Tired of your PC slowing you down? Can’t keep your machine on 24/7 for scrapin
```python ```python
from scrapling.fetchers import Fetcher from scrapling.fetchers import Fetcher
# Do HTTP GET request to a web page and create an Adaptor instance # Do HTTP GET request to a web page and create an Selector instance
page = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True) page = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)
# Get all text content from all HTML tags in the page except the `script` and `style` tags # Get all text content from all HTML tags in the page except the `script` and `style` tags
page.get_all_text(ignore_tags=('script', 'style')) page.get_all_text(ignore_tags=('script', 'style'))
@@ -219,7 +219,7 @@ Here are the results:
| Scrapling | 2.51 | 1.0x | | Scrapling | 2.51 | 1.0x |
| AutoScraper | 11.41 | 4.546x | | AutoScraper | 11.41 | 4.546x |
Scrapling can find elements with more methods and returns the entire element's `Adaptor` object, not only text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them. Scrapling can find elements with more methods and returns the entire element's `Selector` object, not only text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them.
As you see, Scrapling is still 4.5 times faster at the same task. As you see, Scrapling is still 4.5 times faster at the same task.
benchmarks.py
+4 -4
View File
@@ -12,7 +12,7 @@ from parsel import Selector
from pyquery import PyQuery as pq from pyquery import PyQuery as pq
from selectolax.parser import HTMLParser from selectolax.parser import HTMLParser
from scrapling import Adaptor from scrapling import Selector as ScraplingSelector
large_html = ( large_html = (
"<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>" "<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>"
@@ -73,9 +73,9 @@ def test_pyquery():
@benchmark @benchmark
def test_scrapling(): def test_scrapling():
# No need to do `.extract()` like parsel to extract text # No need to do `.extract()` like parsel to extract text
# Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]` # Also, this is faster than `[t.text for t in Selector(large_html, adaptive=False).css('.item')]`
# for obvious reasons, of course. # for obvious reasons, of course.
return Adaptor(large_html, auto_match=False).css(".item::text") return ScraplingSelector(large_html, adaptive=False).css(".item::text")
@benchmark @benchmark
@@ -112,7 +112,7 @@ def test_scrapling_text(request_html):
# Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster # Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
return [ return [
element.text element.text
for element in Adaptor(request_html, auto_match=False) for element in ScraplingSelector(request_html, adaptive=False)
.find_by_text("Tipping the Velvet", first_match=True) .find_by_text("Tipping the Velvet", first_match=True)
.find_similar(ignore_attributes=["title"]) .find_similar(ignore_attributes=["title"])
] ]
scrapling/__init__.py
+5 -5
View File
@@ -10,12 +10,12 @@ def __getattr__(name):
from scrapling.fetchers import Fetcher as cls from scrapling.fetchers import Fetcher as cls
return cls return cls
elif name == "Adaptor": elif name == "Selector":
from scrapling.parser import Adaptor as cls from scrapling.parser import Selector as cls
return cls return cls
elif name == "Adaptors": elif name == "Selectors":
from scrapling.parser import Adaptors as cls from scrapling.parser import Selectors as cls
return cls return cls
elif name == "AttributesHandler": elif name == "AttributesHandler":
@@ -46,4 +46,4 @@ def __getattr__(name):
raise AttributeError(f"module 'scrapling' has no attribute '{name}'") raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
__all__ = ["Adaptor", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"] __all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
scrapling/core/ai.py
+6 -6
View File
@@ -430,7 +430,7 @@ class ScraplingMCPServer:
os_randomize: bool = False, os_randomize: bool = False,
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
) -> ResponseModel: ) -> ResponseModel:
"""Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result. """Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.
Note: This is best suitable for high protection levels. It's slower than the other tools. Note: This is best suitable for high protection levels. It's slower than the other tools.
@@ -467,7 +467,7 @@ class ScraplingMCPServer:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
""" """
page = await StealthyFetcher.async_fetch( page = await StealthyFetcher.async_fetch(
url, url,
@@ -491,7 +491,7 @@ class ScraplingMCPServer:
solve_cloudflare=solve_cloudflare, solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
additional_arguments=additional_arguments, additional_args=additional_args,
) )
return _ContentTranslator( return _ContentTranslator(
Convertor._extract_content( Convertor._extract_content(
@@ -530,7 +530,7 @@ class ScraplingMCPServer:
os_randomize: bool = False, os_randomize: bool = False,
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
) -> List[ResponseModel]: ) -> List[ResponseModel]:
"""Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result. """Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.
Note: This is best suitable for high protection levels. It's slower than the other tools. Note: This is best suitable for high protection levels. It's slower than the other tools.
@@ -567,7 +567,7 @@ class ScraplingMCPServer:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
""" """
async with AsyncStealthySession( async with AsyncStealthySession(
wait=wait, wait=wait,
@@ -591,7 +591,7 @@ class ScraplingMCPServer:
solve_cloudflare=solve_cloudflare, solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
additional_arguments=additional_arguments, additional_args=additional_args,
) as session: ) as session:
tasks = [session.fetch(url) for url in urls] tasks = [session.fetch(url) for url in urls]
responses = await gather(*tasks) responses = await gather(*tasks)
scrapling/core/shell.py
+17 -17
View File
@@ -27,7 +27,7 @@ from orjson import loads as json_loads, JSONDecodeError
from scrapling import __version__ from scrapling import __version__
from scrapling.core.custom_types import TextHandler from scrapling.core.custom_types import TextHandler
from scrapling.core.utils import log from scrapling.core.utils import log
from scrapling.parser import Adaptor, Adaptors from scrapling.parser import Selector, Selectors
from scrapling.core._types import ( from scrapling.core._types import (
List, List,
Optional, Optional,
@@ -399,9 +399,9 @@ class CurlParser:
return None return None
def show_page_in_browser(page: Adaptor): def show_page_in_browser(page: Selector):
if not page or not isinstance(page, Adaptor): if not page or not isinstance(page, Selector):
log.error("Input must be of type `Adaptor`") log.error("Input must be of type `Selector`")
return return
try: try:
@@ -421,7 +421,7 @@ class CustomShell:
def __init__(self, code, log_level="debug"): def __init__(self, code, log_level="debug"):
self.code = code self.code = code
self.page = None self.page = None
self.pages = Adaptors([]) self.pages = Selectors([])
self._curl_parser = CurlParser() self._curl_parser = CurlParser()
log_level = log_level.strip().lower() log_level = log_level.strip().lower()
@@ -457,7 +457,7 @@ class CustomShell:
- Fetcher/AsyncFetcher - Fetcher/AsyncFetcher
- DynamicFetcher - DynamicFetcher
- StealthyFetcher - StealthyFetcher
- Adaptor - Selector
-> Useful shortcuts: -> Useful shortcuts:
- {"get":<30} Shortcut for `Fetcher.get` - {"get":<30} Shortcut for `Fetcher.get`
@@ -469,7 +469,7 @@ class CustomShell:
-> Useful commands -> Useful commands
- {"page / response":<30} The response object of the last page you fetched - {"page / response":<30} The response object of the last page you fetched
- {"pages":<30} Adaptors object of the last 5 response objects you fetched - {"pages":<30} Selectors object of the last 5 response objects you fetched
- {"uncurl('curl_command')":<30} Convert curl command to a Request object. (Optimized to handle curl commands copied from DevTools network tab.) - {"uncurl('curl_command')":<30} Convert curl command to a Request object. (Optimized to handle curl commands copied from DevTools network tab.)
- {"curl2fetcher('curl_command')":<30} Convert curl command and make the request with Fetcher. (Optimized to handle curl commands copied from DevTools network tab.) - {"curl2fetcher('curl_command')":<30} Convert curl command and make the request with Fetcher. (Optimized to handle curl commands copied from DevTools network tab.)
- {"view(page)":<30} View page in a browser - {"view(page)":<30} View page in a browser
@@ -481,7 +481,7 @@ Type 'exit' or press Ctrl+D to exit.
def update_page(self, result): def update_page(self, result):
"""Update the current page and add to pages history""" """Update the current page and add to pages history"""
self.page = result self.page = result
if isinstance(result, (Response, Adaptor)): if isinstance(result, (Response, Selector)):
self.pages.append(result) self.pages.append(result)
if len(self.pages) > 5: if len(self.pages) > 5:
self.pages.pop(0) # Remove oldest item self.pages.pop(0) # Remove oldest item
@@ -528,7 +528,7 @@ Type 'exit' or press Ctrl+D to exit.
"DynamicFetcher": DynamicFetcher, "DynamicFetcher": DynamicFetcher,
"stealthy_fetch": stealthy_fetch, "stealthy_fetch": stealthy_fetch,
"StealthyFetcher": StealthyFetcher, "StealthyFetcher": StealthyFetcher,
"Adaptor": Adaptor, "Selector": Selector,
"page": self.page, "page": self.page,
"response": self.page, "response": self.page,
"pages": self.pages, "pages": self.pages,
@@ -586,14 +586,14 @@ class Convertor:
@classmethod @classmethod
def _extract_content( def _extract_content(
cls, cls,
page: Adaptor, page: Selector,
extraction_type: extraction_types = "markdown", extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None, css_selector: Optional[str] = None,
main_content_only: bool = False, main_content_only: bool = False,
) -> Generator[str, None, None]: ) -> Generator[str, None, None]:
"""Extract the content of an Adaptor""" """Extract the content of an Selector"""
if not page or not isinstance(page, Adaptor): if not page or not isinstance(page, Selector):
raise TypeError("Input must be of type `Adaptor`") raise TypeError("Input must be of type `Selector`")
elif not extraction_type or extraction_type not in cls._extension_map.values(): elif not extraction_type or extraction_type not in cls._extension_map.values():
raise ValueError(f"Unknown extraction type: {extraction_type}") raise ValueError(f"Unknown extraction type: {extraction_type}")
else: else:
@@ -622,11 +622,11 @@ class Convertor:
@classmethod @classmethod
def write_content_to_file( def write_content_to_file(
cls, page: Adaptor, filename: str, css_selector: Optional[str] = None cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None: ) -> None:
"""Write an Adaptor's content to a file""" """Write an Selector's content to a file"""
if not page or not isinstance(page, Adaptor): if not page or not isinstance(page, Selector):
raise TypeError("Input must be of type `Adaptor`") raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip(): elif not filename or not isinstance(filename, str) or not filename.strip():
raise ValueError("Filename must be provided") raise ValueError("Filename must be provided")
elif not filename.endswith((".md", ".html", ".txt")): elif not filename.endswith((".md", ".html", ".txt")):
scrapling/core/storage_adaptors.py → scrapling/core/storage.py
View File
scrapling/engines/_browsers/_camoufox.py
+20 -20
View File
@@ -70,8 +70,8 @@ class StealthySession:
"os_randomize", "os_randomize",
"disable_ads", "disable_ads",
"geoip", "geoip",
"adaptor_arguments", "selector_config",
"additional_arguments", "additional_args",
"playwright", "playwright",
"browser", "browser",
"context", "context",
@@ -105,8 +105,8 @@ class StealthySession:
os_randomize: bool = False, os_randomize: bool = False,
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
): ):
"""A Browser session manager with page pooling """A Browser session manager with page pooling
@@ -136,8 +136,8 @@ class StealthySession:
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
""" """
params = { params = {
@@ -163,8 +163,8 @@ class StealthySession:
"os_randomize": os_randomize, "os_randomize": os_randomize,
"disable_ads": disable_ads, "disable_ads": disable_ads,
"geoip": geoip, "geoip": geoip,
"adaptor_arguments": adaptor_arguments, "selector_config": selector_config,
"additional_arguments": additional_arguments, "additional_args": additional_args,
} }
config = validate(params, CamoufoxConfig) config = validate(params, CamoufoxConfig)
@@ -190,14 +190,14 @@ class StealthySession:
self.os_randomize = config.os_randomize self.os_randomize = config.os_randomize
self.disable_ads = config.disable_ads self.disable_ads = config.disable_ads
self.geoip = config.geoip self.geoip = config.geoip
self.adaptor_arguments = config.adaptor_arguments self.selector_config = config.selector_config
self.additional_arguments = config.additional_arguments self.additional_args = config.additional_args
self.playwright: Optional[Playwright] = None self.playwright: Optional[Playwright] = None
self.context: Optional[BrowserContext] = None self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages) self.page_pool = PagePool(self.max_pages)
self._closed = False self._closed = False
self.adaptor_arguments = config.adaptor_arguments self.selector_config = config.selector_config
self.page_action = config.page_action self.page_action = config.page_action
self._headers_keys = ( self._headers_keys = (
set(map(str.lower, self.extra_headers.keys())) set(map(str.lower, self.extra_headers.keys()))
@@ -223,7 +223,7 @@ class StealthySession:
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self.os_randomize else get_os_name(), "os": None if self.os_randomize else get_os_name(),
"user_data_dir": "", "user_data_dir": "",
**self.additional_arguments, **self.additional_args,
} }
) )
@@ -433,7 +433,7 @@ class StealthySession:
page_info.page.wait_for_timeout(self.wait) page_info.page.wait_for_timeout(self.wait)
response = ResponseFactory.from_playwright_response( response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments page_info.page, first_response, final_response, self.selector_config
) )
# Mark the page as ready for next use # Mark the page as ready for next use
@@ -482,8 +482,8 @@ class AsyncStealthySession(StealthySession):
os_randomize: bool = False, os_randomize: bool = False,
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
): ):
"""A Browser session manager with page pooling """A Browser session manager with page pooling
@@ -513,8 +513,8 @@ class AsyncStealthySession(StealthySession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
""" """
super().__init__( super().__init__(
max_pages, max_pages,
@@ -539,8 +539,8 @@ class AsyncStealthySession(StealthySession):
os_randomize, os_randomize,
disable_ads, disable_ads,
geoip, geoip,
adaptor_arguments, selector_config,
additional_arguments, additional_args,
) )
self.playwright: Optional[AsyncPlaywright] = None self.playwright: Optional[AsyncPlaywright] = None
self.context: Optional[AsyncBrowserContext] = None self.context: Optional[AsyncBrowserContext] = None
@@ -731,7 +731,7 @@ class AsyncStealthySession(StealthySession):
# Create response object # Create response object
response = await ResponseFactory.from_async_playwright_response( response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments page_info.page, first_response, final_response, self.selector_config
) )
# Mark the page as ready for next use # Mark the page as ready for next use
scrapling/engines/_browsers/_controllers.py
+10 -10
View File
@@ -70,7 +70,7 @@ class DynamicSession:
"context", "context",
"page_pool", "page_pool",
"_closed", "_closed",
"adaptor_arguments", "selector_config",
"page_action", "page_action",
"launch_options", "launch_options",
"context_options", "context_options",
@@ -100,7 +100,7 @@ class DynamicSession:
cookies: Optional[List[Dict]] = None, cookies: Optional[List[Dict]] = None,
network_idle: bool = False, network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached", wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
): ):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
@@ -125,7 +125,7 @@ class DynamicSession:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
""" """
params = { params = {
@@ -143,7 +143,7 @@ class DynamicSession:
"extra_headers": extra_headers, "extra_headers": extra_headers,
"useragent": useragent, "useragent": useragent,
"timeout": timeout, "timeout": timeout,
"adaptor_arguments": adaptor_arguments, "selector_config": selector_config,
"disable_resources": disable_resources, "disable_resources": disable_resources,
"wait_selector": wait_selector, "wait_selector": wait_selector,
"cookies": cookies, "cookies": cookies,
@@ -177,7 +177,7 @@ class DynamicSession:
self.context: Optional[BrowserContext] = None self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages) self.page_pool = PagePool(self.max_pages)
self._closed = False self._closed = False
self.adaptor_arguments = config.adaptor_arguments self.selector_config = config.selector_config
self.page_action = config.page_action self.page_action = config.page_action
self._headers_keys = ( self._headers_keys = (
set(map(str.lower, self.extra_headers.keys())) set(map(str.lower, self.extra_headers.keys()))
@@ -370,7 +370,7 @@ class DynamicSession:
# Create response object # Create response object
response = ResponseFactory.from_playwright_response( response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments page_info.page, first_response, final_response, self.selector_config
) )
# Mark the page as ready for next use # Mark the page as ready for next use
@@ -417,7 +417,7 @@ class AsyncDynamicSession(DynamicSession):
cookies: Optional[List[Dict]] = None, cookies: Optional[List[Dict]] = None,
network_idle: bool = False, network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached", wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
): ):
"""A Browser session manager with page pooling """A Browser session manager with page pooling
@@ -443,7 +443,7 @@ class AsyncDynamicSession(DynamicSession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
""" """
super().__init__( super().__init__(
@@ -467,7 +467,7 @@ class AsyncDynamicSession(DynamicSession):
cookies, cookies,
network_idle, network_idle,
wait_selector_state, wait_selector_state,
adaptor_arguments, selector_config,
) )
self.playwright: Optional[AsyncPlaywright] = None self.playwright: Optional[AsyncPlaywright] = None
@@ -623,7 +623,7 @@ class AsyncDynamicSession(DynamicSession):
# Create response object # Create response object
response = await ResponseFactory.from_async_playwright_response( response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.adaptor_arguments page_info.page, first_response, final_response, self.selector_config
) )
# Mark the page as ready for next use # Mark the page as ready for next use
scrapling/engines/_browsers/_validators.py
+9 -9
View File
@@ -39,7 +39,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
cookies: Optional[List[Dict]] = None cookies: Optional[List[Dict]] = None
network_idle: bool = False network_idle: bool = False
wait_selector_state: SelectorWaitStates = "attached" wait_selector_state: SelectorWaitStates = "attached"
adaptor_arguments: Optional[Dict] = None selector_config: Optional[Dict] = None
def __post_init__(self): def __post_init__(self):
"""Custom validation after msgspec validation""" """Custom validation after msgspec validation"""
@@ -57,8 +57,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
self.__validate_cdp(self.cdp_url) self.__validate_cdp(self.cdp_url)
if not self.cookies: if not self.cookies:
self.cookies = [] self.cookies = []
if not self.adaptor_arguments: if not self.selector_config:
self.adaptor_arguments = {} self.selector_config = {}
@staticmethod @staticmethod
def __validate_cdp(cdp_url): def __validate_cdp(cdp_url):
@@ -105,8 +105,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
os_randomize: bool = False os_randomize: bool = False
disable_ads: bool = False disable_ads: bool = False
geoip: bool = False geoip: bool = False
adaptor_arguments: Optional[Dict] = None selector_config: Optional[Dict] = None
additional_arguments: Optional[Dict] = None additional_args: Optional[Dict] = None
def __post_init__(self): def __post_init__(self):
"""Custom validation after msgspec validation""" """Custom validation after msgspec validation"""
@@ -136,10 +136,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.cookies = [] self.cookies = []
if self.solve_cloudflare and self.timeout < 60_000: if self.solve_cloudflare and self.timeout < 60_000:
self.timeout = 60_000 self.timeout = 60_000
if not self.adaptor_arguments: if not self.selector_config:
self.adaptor_arguments = {} self.selector_config = {}
if not self.additional_arguments: if not self.additional_args:
self.additional_arguments = {} self.additional_args = {}
def validate(params, model): def validate(params, model):
scrapling/engines/static.py
+12 -18
View File
@@ -63,7 +63,7 @@ class FetcherSession:
max_redirects: int = 30, max_redirects: int = 30,
verify: bool = True, verify: bool = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
): ):
""" """
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
@@ -81,7 +81,7 @@ class FetcherSession:
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class. :param selector_config: Arguments passed when creating the final Selector class.
""" """
self.default_impersonate = impersonate self.default_impersonate = impersonate
self.stealth = stealthy_headers self.stealth = stealthy_headers
@@ -97,7 +97,7 @@ class FetcherSession:
self.default_verify = verify self.default_verify = verify
self.default_cert = cert self.default_cert = cert
self.default_http3 = http3 self.default_http3 = http3
self.adaptor_arguments = adaptor_arguments or {} self.selector_config = selector_config or {}
self._curl_session: Optional[CurlSession] = None self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] = None self._async_curl_session: Optional[AsyncCurlSession] = None
@@ -260,7 +260,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -270,7 +270,7 @@ class FetcherSession:
:param request_args: Arguments to be passed to the session's `request()` method. :param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request. :param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries. :param retry_delay: Number of seconds to wait between retries.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class. :param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous. :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
""" """
session = self._curl_session session = self._curl_session
@@ -286,9 +286,7 @@ class FetcherSession:
try: try:
response = session.request(method, **request_args) response = session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400 # response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request( return ResponseFactory.from_http_request(response, selector_config)
response, adaptor_arguments
)
except CurlError as e: except CurlError as e:
if attempt < max_retries - 1: if attempt < max_retries - 1:
log.error( log.error(
@@ -307,7 +305,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
adaptor_arguments: Optional[Dict] = None, selector_config: Optional[Dict] = None,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -317,7 +315,7 @@ class FetcherSession:
:param request_args: Arguments to be passed to the session's `request()` method. :param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request. :param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries. :param retry_delay: Number of seconds to wait between retries.
:param adaptor_arguments: Arguments passed when creating the final Adaptor class. :param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous. :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
""" """
session = self._async_curl_session session = self._async_curl_session
@@ -335,9 +333,7 @@ class FetcherSession:
try: try:
response = await session.request(method, **request_args) response = await session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400 # response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request( return ResponseFactory.from_http_request(response, selector_config)
response, adaptor_arguments
)
except CurlError as e: except CurlError as e:
if attempt < max_retries - 1: if attempt < max_retries - 1:
log.error( log.error(
@@ -373,9 +369,7 @@ class FetcherSession:
""" """
stealth = self.stealth if stealth is None else stealth stealth = self.stealth if stealth is None else stealth
adaptor_arguments = ( selector_config = kwargs.pop("selector_config", {}) or self.selector_config
kwargs.pop("adaptor_arguments", {}) or self.adaptor_arguments
)
max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries)
retry_delay = self.get_with_precedence( retry_delay = self.get_with_precedence(
kwargs, "retry_delay", self.default_retry_delay kwargs, "retry_delay", self.default_retry_delay
@@ -383,12 +377,12 @@ class FetcherSession:
request_args = self._merge_request_args(stealth=stealth, **kwargs) request_args = self._merge_request_args(stealth=stealth, **kwargs)
if self._curl_session: if self._curl_session:
return self.__make_request( return self.__make_request(
method, request_args, max_retries, retry_delay, adaptor_arguments method, request_args, max_retries, retry_delay, selector_config
) )
elif self._async_curl_session: elif self._async_curl_session:
# The returned value is a Coroutine # The returned value is a Coroutine
return self.__make_async_request( return self.__make_async_request(
method, request_args, max_retries, retry_delay, adaptor_arguments method, request_args, max_retries, retry_delay, selector_config
) )
raise RuntimeError("No active session available.") raise RuntimeError("No active session available.")
scrapling/engines/toolbelt/convertor.py
+1 -1
View File
@@ -239,7 +239,7 @@ class ResponseFactory:
:param response: `curl_cffi` response object :param response: `curl_cffi` response object
:param parser_arguments: Additional arguments to be passed to the `Response` object constructor. :param parser_arguments: Additional arguments to be passed to the `Response` object constructor.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
return Response( return Response(
url=response.url, url=response.url,
scrapling/engines/toolbelt/custom.py
+20 -22
View File
@@ -15,7 +15,7 @@ from scrapling.core._types import (
) )
from scrapling.core.custom_types import MappingProxyType from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem from scrapling.parser import Selector, SQLiteStorageSystem
class ResponseEncoding: class ResponseEncoding:
@@ -97,7 +97,7 @@ class ResponseEncoding:
return cls.__DEFAULT_ENCODING return cls.__DEFAULT_ENCODING
class Response(Adaptor): class Response(Selector):
"""This class is returned by all engines as a way to unify response type between different libraries.""" """This class is returned by all engines as a way to unify response type between different libraries."""
def __init__( def __init__(
@@ -113,9 +113,9 @@ class Response(Adaptor):
encoding: str = "utf-8", encoding: str = "utf-8",
method: str = "GET", method: str = "GET",
history: List = None, history: List = None,
**adaptor_arguments: Dict, **selector_config: Dict,
): ):
automatch_domain = adaptor_arguments.pop("automatch_domain", None) adaptive_domain = selector_config.pop("adaptive_domain", None)
self.status = status self.status = status
self.reason = reason self.reason = reason
self.cookies = cookies self.cookies = cookies
@@ -126,12 +126,10 @@ class Response(Adaptor):
super().__init__( super().__init__(
text=text, text=text,
body=body, body=body,
url=automatch_domain or url, url=adaptive_domain or url,
encoding=encoding, encoding=encoding,
**adaptor_arguments, **selector_config,
) )
# For backward compatibility
self.adaptor = self
# For easier debugging while working from a Python shell # For easier debugging while working from a Python shell
log.info( log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})" f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
@@ -144,20 +142,20 @@ class Response(Adaptor):
class BaseFetcher: class BaseFetcher:
__slots__ = () __slots__ = ()
huge_tree: bool = True huge_tree: bool = True
auto_match: Optional[bool] = False adaptive: Optional[bool] = False
storage: Any = SQLiteStorageSystem storage: Any = SQLiteStorageSystem
keep_cdata: Optional[bool] = False keep_cdata: Optional[bool] = False
storage_args: Optional[Dict] = None storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None adaptive_domain: Optional[str] = None
parser_keywords: Tuple = ( parser_keywords: Tuple = (
"huge_tree", "huge_tree",
"auto_match", "adaptive",
"storage", "storage",
"keep_cdata", "keep_cdata",
"storage_args", "storage_args",
"keep_comments", "keep_comments",
"automatch_domain", "adaptive_domain",
) # Left open for the user ) # Left open for the user
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@@ -178,17 +176,17 @@ class BaseFetcher:
huge_tree=cls.huge_tree, huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments, keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata, keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match, adaptive=cls.adaptive,
storage=cls.storage, storage=cls.storage,
storage_args=cls.storage_args, storage_args=cls.storage_args,
automatch_domain=cls.automatch_domain, adaptive_domain=cls.adaptive_domain,
) )
@classmethod @classmethod
def configure(cls, **kwargs): def configure(cls, **kwargs):
"""Set multiple arguments for the parser at once globally """Set multiple arguments for the parser at once globally
:param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, adaptive, storage, storage_args, adaptive_domain
""" """
for key, value in kwargs.items(): for key, value in kwargs.items():
key = key.strip().lower() key = key.strip().lower()
@@ -212,23 +210,23 @@ class BaseFetcher:
@classmethod @classmethod
def _generate_parser_arguments(cls) -> Dict: def _generate_parser_arguments(cls) -> Dict:
# Adaptor class parameters # Selector class parameters
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later # I won't validate Selector's class parameters here again, I will leave it to be validated later
parser_arguments = dict( parser_arguments = dict(
huge_tree=cls.huge_tree, huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments, keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata, keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match, adaptive=cls.adaptive,
storage=cls.storage, storage=cls.storage,
storage_args=cls.storage_args, storage_args=cls.storage_args,
) )
if cls.automatch_domain: if cls.adaptive_domain:
if type(cls.automatch_domain) is not str: if type(cls.adaptive_domain) is not str:
log.warning( log.warning(
'[Ignored] The argument "automatch_domain" must be of string type' '[Ignored] The argument "adaptive_domain" must be of string type'
) )
else: else:
parser_arguments.update({"automatch_domain": cls.automatch_domain}) parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
return parser_arguments return parser_arguments
scrapling/fetchers.py
+10 -10
View File
@@ -74,7 +74,7 @@ class StealthyFetcher(BaseFetcher):
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
custom_config: Optional[Dict] = None, custom_config: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -106,7 +106,7 @@ class StealthyFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object. :return: A `Response` object.
""" """
if not custom_config: if not custom_config:
@@ -139,8 +139,8 @@ class StealthyFetcher(BaseFetcher):
solve_cloudflare=solve_cloudflare, solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, selector_config={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {}, additional_args=additional_args or {},
) as engine: ) as engine:
return engine.fetch(url) return engine.fetch(url)
@@ -170,7 +170,7 @@ class StealthyFetcher(BaseFetcher):
disable_ads: bool = False, disable_ads: bool = False,
geoip: bool = False, geoip: bool = False,
custom_config: Optional[Dict] = None, custom_config: Optional[Dict] = None,
additional_arguments: Optional[Dict] = None, additional_args: Optional[Dict] = None,
) -> Response: ) -> Response:
""" """
Opens up a browser and do your request based on your chosen options below. Opens up a browser and do your request based on your chosen options below.
@@ -202,7 +202,7 @@ class StealthyFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object. :return: A `Response` object.
""" """
if not custom_config: if not custom_config:
@@ -235,8 +235,8 @@ class StealthyFetcher(BaseFetcher):
solve_cloudflare=solve_cloudflare, solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, selector_config={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {}, additional_args=additional_args or {},
) as engine: ) as engine:
return await engine.fetch(url) return await engine.fetch(url)
@@ -337,7 +337,7 @@ class DynamicFetcher(BaseFetcher):
disable_webgl=disable_webgl, disable_webgl=disable_webgl,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, selector_config={**cls._generate_parser_arguments(), **custom_config},
) as session: ) as session:
return session.fetch(url) return session.fetch(url)
@@ -421,7 +421,7 @@ class DynamicFetcher(BaseFetcher):
disable_webgl=disable_webgl, disable_webgl=disable_webgl,
disable_resources=disable_resources, disable_resources=disable_resources,
wait_selector_state=wait_selector_state, wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config}, selector_config={**cls._generate_parser_arguments(), **custom_config},
) as session: ) as session:
return await session.fetch(url) return await session.fetch(url)
scrapling/parser.py
+105 -107
View File
@@ -24,7 +24,7 @@ from scrapling.core._types import (
) )
from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers
from scrapling.core.mixins import SelectorsGeneration from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.storage_adaptors import ( from scrapling.core.storage import (
SQLiteStorageSystem, SQLiteStorageSystem,
StorageSystemMixin, StorageSystemMixin,
_StorageTools, _StorageTools,
@@ -33,11 +33,11 @@ from scrapling.core.translator import translator_instance
from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log
class Adaptor(SelectorsGeneration): class Selector(SelectorsGeneration):
__slots__ = ( __slots__ = (
"url", "url",
"encoding", "encoding",
"__auto_match_enabled", "__adaptive_enabled",
"_root", "_root",
"_storage", "_storage",
"__keep_comments", "__keep_comments",
@@ -58,7 +58,7 @@ class Adaptor(SelectorsGeneration):
root: Optional[html.HtmlElement] = None, root: Optional[html.HtmlElement] = None,
keep_comments: Optional[bool] = False, keep_comments: Optional[bool] = False,
keep_cdata: Optional[bool] = False, keep_cdata: Optional[bool] = False,
auto_match: Optional[bool] = False, adaptive: Optional[bool] = False,
_storage: object = None, _storage: object = None,
storage: Any = SQLiteStorageSystem, storage: Any = SQLiteStorageSystem,
storage_args: Optional[Dict] = None, storage_args: Optional[Dict] = None,
@@ -82,7 +82,7 @@ class Adaptor(SelectorsGeneration):
Don't use it unless you know what you are doing! Don't use it unless you know what you are doing!
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
:param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML. :param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
:param auto_match: Globally turn off the auto-match feature in all functions, this argument takes higher :param adaptive: Globally turn off the auto-match feature in all functions, this argument takes higher
priority over all auto-match related arguments/functions in the class. priority over all auto-match related arguments/functions in the class.
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info. :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
:param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class. :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
@@ -90,7 +90,7 @@ class Adaptor(SelectorsGeneration):
""" """
if root is None and not body and text is None: if root is None and not body and text is None:
raise ValueError( raise ValueError(
"Adaptor class needs text, body, or root arguments to work" "Selector class needs text, body, or root arguments to work"
) )
self.__text = "" self.__text = ""
@@ -134,9 +134,9 @@ class Adaptor(SelectorsGeneration):
self._root = root self._root = root
self.__auto_match_enabled = auto_match self.__adaptive_enabled = adaptive
if self.__auto_match_enabled: if self.__adaptive_enabled:
if _storage is not None: if _storage is not None:
self._storage = _storage self._storage = _storage
else: else:
@@ -214,17 +214,17 @@ class Adaptor(SelectorsGeneration):
""" """
return TextHandler(str(element)) return TextHandler(str(element))
def __element_convertor(self, element: html.HtmlElement) -> "Adaptor": def __element_convertor(self, element: html.HtmlElement) -> "Selector":
"""Used internally to convert a single HtmlElement to Adaptor directly without checks""" """Used internally to convert a single HtmlElement to Selector directly without checks"""
db_instance = ( db_instance = (
self._storage if (hasattr(self, "_storage") and self._storage) else None self._storage if (hasattr(self, "_storage") and self._storage) else None
) )
return Adaptor( return Selector(
root=element, root=element,
url=self.url, url=self.url,
encoding=self.encoding, encoding=self.encoding,
auto_match=self.__auto_match_enabled, adaptive=self.__adaptive_enabled,
_storage=db_instance, # Reuse existing storage if it exists otherwise it won't be checked if `auto_match` is turned off _storage=db_instance, # Reuse existing storage if it exists otherwise it won't be checked if `adaptive` is turned off
keep_comments=self.__keep_comments, keep_comments=self.__keep_comments,
keep_cdata=self.__keep_cdata, keep_cdata=self.__keep_cdata,
huge_tree=self.__huge_tree_enabled, huge_tree=self.__huge_tree_enabled,
@@ -233,8 +233,8 @@ class Adaptor(SelectorsGeneration):
def __handle_element( def __handle_element(
self, element: Union[html.HtmlElement, etree._ElementUnicodeResult] self, element: Union[html.HtmlElement, etree._ElementUnicodeResult]
) -> Union[TextHandler, "Adaptor", None]: ) -> Union[TextHandler, "Selector", None]:
"""Used internally in all functions to convert a single element to type (Adaptor|TextHandler) when possible""" """Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible"""
if element is None: if element is None:
return None return None
elif self._is_text_node(element): elif self._is_text_node(element):
@@ -245,23 +245,23 @@ class Adaptor(SelectorsGeneration):
def __handle_elements( def __handle_elements(
self, result: List[Union[html.HtmlElement, etree._ElementUnicodeResult]] self, result: List[Union[html.HtmlElement, etree._ElementUnicodeResult]]
) -> Union["Adaptors", "TextHandlers", List]: ) -> Union["Selectors", "TextHandlers", List]:
"""Used internally in all functions to convert results to type (Adaptors|TextHandlers) in bulk when possible""" """Used internally in all functions to convert results to type (Selectors|TextHandlers) in bulk when possible"""
if not len( if not len(
result result
): # Lxml will give a warning if I used something like `not result` ): # Lxml will give a warning if I used something like `not result`
return Adaptors([]) return Selectors([])
# From within the code, this method will always get a list of the same type, # From within the code, this method will always get a list of the same type,
# so we will continue without checks for a slight performance boost # so we will continue without checks for a slight performance boost
if self._is_text_node(result[0]): if self._is_text_node(result[0]):
return TextHandlers(list(map(self.__content_convertor, result))) return TextHandlers(list(map(self.__content_convertor, result)))
return Adaptors(list(map(self.__element_convertor, result))) return Selectors(list(map(self.__element_convertor, result)))
def __getstate__(self) -> Any: def __getstate__(self) -> Any:
# lxml don't like it :) # lxml don't like it :)
raise TypeError("Can't pickle Adaptor objects") raise TypeError("Can't pickle Selector objects")
# The following four properties I made them into functions instead of variables directly # The following four properties I made them into functions instead of variables directly
# So they don't slow down the process of initializing many instances of the class and gets executed only # So they don't slow down the process of initializing many instances of the class and gets executed only
@@ -322,7 +322,7 @@ class Adaptor(SelectorsGeneration):
return TextHandler(separator).join(_all_strings) return TextHandler(separator).join(_all_strings)
def urljoin(self, relative_url: str) -> str: def urljoin(self, relative_url: str) -> str:
"""Join this Adaptor's url with a relative url to form an absolute full URL.""" """Join this Selector's url with a relative url to form an absolute full URL."""
return urljoin(self.url, relative_url) return urljoin(self.url, relative_url)
@property @property
@@ -363,20 +363,20 @@ class Adaptor(SelectorsGeneration):
return class_name in self._root.classes return class_name in self._root.classes
@property @property
def parent(self) -> Union["Adaptor", None]: def parent(self) -> Union["Selector", None]:
"""Return the direct parent of the element or ``None`` otherwise""" """Return the direct parent of the element or ``None`` otherwise"""
return self.__handle_element(self._root.getparent()) return self.__handle_element(self._root.getparent())
@property @property
def below_elements(self) -> "Adaptors[Adaptor]": def below_elements(self) -> "Selectors[Selector]":
"""Return all elements under the current element in the DOM tree""" """Return all elements under the current element in the DOM tree"""
below = self._root.xpath(".//*") below = self._root.xpath(".//*")
return self.__handle_elements(below) return self.__handle_elements(below)
@property @property
def children(self) -> "Adaptors[Adaptor]": def children(self) -> "Selectors[Selector]":
"""Return the children elements of the current element or empty list otherwise""" """Return the children elements of the current element or empty list otherwise"""
return Adaptors( return Selectors(
[ [
self.__element_convertor(child) self.__element_convertor(child)
for child in self._root.iterchildren() for child in self._root.iterchildren()
@@ -385,22 +385,22 @@ class Adaptor(SelectorsGeneration):
) )
@property @property
def siblings(self) -> "Adaptors[Adaptor]": def siblings(self) -> "Selectors[Selector]":
"""Return other children of the current element's parent or empty list otherwise""" """Return other children of the current element's parent or empty list otherwise"""
if self.parent: if self.parent:
return Adaptors( return Selectors(
[child for child in self.parent.children if child._root != self._root] [child for child in self.parent.children if child._root != self._root]
) )
return Adaptors([]) return Selectors([])
def iterancestors(self) -> Generator["Adaptor", None, None]: def iterancestors(self) -> Generator["Selector", None, None]:
"""Return a generator that loops over all ancestors of the element, starting with the element's parent.""" """Return a generator that loops over all ancestors of the element, starting with the element's parent."""
for ancestor in self._root.iterancestors(): for ancestor in self._root.iterancestors():
yield self.__element_convertor(ancestor) yield self.__element_convertor(ancestor)
def find_ancestor( def find_ancestor(
self, func: Callable[["Adaptor"], bool] self, func: Callable[["Selector"], bool]
) -> Union["Adaptor", None]: ) -> Union["Selector", None]:
"""Loop over all ancestors of the element till one match the passed function """Loop over all ancestors of the element till one match the passed function
:param func: A function that takes each ancestor as an argument and returns True/False :param func: A function that takes each ancestor as an argument and returns True/False
:return: The first ancestor that match the function or ``None`` otherwise. :return: The first ancestor that match the function or ``None`` otherwise.
@@ -411,13 +411,13 @@ class Adaptor(SelectorsGeneration):
return None return None
@property @property
def path(self) -> "Adaptors[Adaptor]": def path(self) -> "Selectors[Selector]":
"""Returns a list of type `Adaptors` that contains the path leading to the current element from the root.""" """Returns a list of type `Selectors` that contains the path leading to the current element from the root."""
lst = list(self.iterancestors()) lst = list(self.iterancestors())
return Adaptors(lst) return Selectors(lst)
@property @property
def next(self) -> Union["Adaptor", None]: def next(self) -> Union["Selector", None]:
"""Returns the next element of the current element in the children of the parent or ``None`` otherwise.""" """Returns the next element of the current element in the children of the parent or ``None`` otherwise."""
next_element = self._root.getnext() next_element = self._root.getnext()
if next_element is not None: if next_element is not None:
@@ -428,7 +428,7 @@ class Adaptor(SelectorsGeneration):
return self.__handle_element(next_element) return self.__handle_element(next_element)
@property @property
def previous(self) -> Union["Adaptor", None]: def previous(self) -> Union["Selector", None]:
"""Returns the previous element of the current element in the children of the parent or ``None`` otherwise.""" """Returns the previous element of the current element in the children of the parent or ``None`` otherwise."""
prev_element = self._root.getprevious() prev_element = self._root.getprevious()
if prev_element is not None: if prev_element is not None:
@@ -471,18 +471,18 @@ class Adaptor(SelectorsGeneration):
# From here we start with the selecting functions # From here we start with the selecting functions
def relocate( def relocate(
self, self,
element: Union[Dict, html.HtmlElement, "Adaptor"], element: Union[Dict, html.HtmlElement, "Selector"],
percentage: int = 0, percentage: int = 0,
adaptor_type: bool = False, selector_type: bool = False,
) -> Union[List[Union[html.HtmlElement, None]], "Adaptors"]: ) -> Union[List[Union[html.HtmlElement, None]], "Selectors"]:
"""This function will search again for the element in the page tree, used automatically on page structure change """This function will search again for the element in the page tree, used automatically on page structure change
:param element: The element we want to relocate in the tree :param element: The element we want to relocate in the tree
:param percentage: The minimum percentage to accept and not going lower than that. Be aware that the percentage :param percentage: The minimum percentage to accept and not going lower than that. Be aware that the percentage
calculation depends solely on the page structure, so don't play with this number unless you must know calculation depends solely on the page structure, so don't play with this number unless you must know
what you are doing! what you are doing!
:param adaptor_type: If True, the return result will be converted to `Adaptors` object :param selector_type: If True, the return result will be converted to `Selectors` object
:return: List of pure HTML elements that got the highest matching score or 'Adaptors' object :return: List of pure HTML elements that got the highest matching score or 'Selectors' object
""" """
score_table = {} score_table = {}
# Note: `element` will most likely always be a dictionary at this point. # Note: `element` will most likely always be a dictionary at this point.
@@ -511,7 +511,7 @@ class Adaptor(SelectorsGeneration):
f"{percent} -> {self.__handle_elements(score_table[percent])}" f"{percent} -> {self.__handle_elements(score_table[percent])}"
) )
if not adaptor_type: if not selector_type:
return score_table[highest_probability] return score_table[highest_probability]
return self.__handle_elements(score_table[highest_probability]) return self.__handle_elements(score_table[highest_probability])
return [] return []
@@ -520,10 +520,10 @@ class Adaptor(SelectorsGeneration):
self, self,
selector: str, selector: str,
identifier: str = "", identifier: str = "",
auto_match: bool = False, adaptive: bool = False,
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
) -> Union["Adaptor", "TextHandler", None]: ) -> Union["Selector", "TextHandler", None]:
"""Search the current tree with CSS3 selectors and return the first result if possible, otherwise return `None` """Search the current tree with CSS3 selectors and return the first result if possible, otherwise return `None`
**Important: **Important:
@@ -531,17 +531,15 @@ class Adaptor(SelectorsGeneration):
and want to relocate the same element(s)** and want to relocate the same element(s)**
:param selector: The CSS3 selector to be used. :param selector: The CSS3 selector to be used.
:param auto_match: Enabled will make the function try to relocate the element if it was 'saved' before :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
:param identifier: A string that will be used to save/retrieve element's data in auto-matching, :param identifier: A string that will be used to save/retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
""" """
for element in self.css( for element in self.css(selector, identifier, adaptive, auto_save, percentage):
selector, identifier, auto_match, auto_save, percentage
):
return element return element
return None return None
@@ -549,11 +547,11 @@ class Adaptor(SelectorsGeneration):
self, self,
selector: str, selector: str,
identifier: str = "", identifier: str = "",
auto_match: bool = False, adaptive: bool = False,
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
**kwargs: Any, **kwargs: Any,
) -> Union["Adaptor", "TextHandler", None]: ) -> Union["Selector", "TextHandler", None]:
"""Search the current tree with XPath selectors and return the first result if possible, otherwise return `None` """Search the current tree with XPath selectors and return the first result if possible, otherwise return `None`
**Important: **Important:
@@ -563,16 +561,16 @@ class Adaptor(SelectorsGeneration):
Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!** Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**
:param selector: The XPath selector to be used. :param selector: The XPath selector to be used.
:param auto_match: Enabled will make the function try to relocate the element if it was 'saved' before :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
:param identifier: A string that will be used to save/retrieve element's data in auto-matching, :param identifier: A string that will be used to save/retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
""" """
for element in self.xpath( for element in self.xpath(
selector, identifier, auto_match, auto_save, percentage, **kwargs selector, identifier, adaptive, auto_save, percentage, **kwargs
): ):
return element return element
return None return None
@@ -581,10 +579,10 @@ class Adaptor(SelectorsGeneration):
self, self,
selector: str, selector: str,
identifier: str = "", identifier: str = "",
auto_match: bool = False, adaptive: bool = False,
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
) -> Union["Adaptors[Adaptor]", List, "TextHandlers[TextHandler]"]: ) -> Union["Selectors[Selector]", List, "TextHandlers[TextHandler]"]:
"""Search the current tree with CSS3 selectors """Search the current tree with CSS3 selectors
**Important: **Important:
@@ -592,24 +590,24 @@ class Adaptor(SelectorsGeneration):
and want to relocate the same element(s)** and want to relocate the same element(s)**
:param selector: The CSS3 selector to be used. :param selector: The CSS3 selector to be used.
:param auto_match: Enabled will make the function try to relocate the element if it was 'saved' before :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
:param identifier: A string that will be used to save/retrieve element's data in auto-matching, :param identifier: A string that will be used to save/retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: `Adaptors` class. :return: `Selectors` class.
""" """
try: try:
if not self.__auto_match_enabled or "," not in selector: if not self.__adaptive_enabled or "," not in selector:
# No need to split selectors in this case, let's save some CPU cycles :) # No need to split selectors in this case, let's save some CPU cycles :)
xpath_selector = translator_instance.css_to_xpath(selector) xpath_selector = translator_instance.css_to_xpath(selector)
return self.xpath( return self.xpath(
xpath_selector, xpath_selector,
identifier or selector, identifier or selector,
auto_match, adaptive,
auto_save, auto_save,
percentage, percentage,
) )
@@ -625,7 +623,7 @@ class Adaptor(SelectorsGeneration):
results += self.xpath( results += self.xpath(
xpath_selector, xpath_selector,
identifier or single_selector.canonical(), identifier or single_selector.canonical(),
auto_match, adaptive,
auto_save, auto_save,
percentage, percentage,
) )
@@ -643,11 +641,11 @@ class Adaptor(SelectorsGeneration):
self, self,
selector: str, selector: str,
identifier: str = "", identifier: str = "",
auto_match: bool = False, adaptive: bool = False,
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
**kwargs: Any, **kwargs: Any,
) -> Union["Adaptors[Adaptor]", List, "TextHandlers[TextHandler]"]: ) -> Union["Selectors[Selector]", List, "TextHandlers[TextHandler]"]:
"""Search the current tree with XPath selectors """Search the current tree with XPath selectors
**Important: **Important:
@@ -657,31 +655,31 @@ class Adaptor(SelectorsGeneration):
Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!** Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**
:param selector: The XPath selector to be used. :param selector: The XPath selector to be used.
:param auto_match: Enabled will make the function try to relocate the element if it was 'saved' before :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
:param identifier: A string that will be used to save/retrieve element's data in auto-matching, :param identifier: A string that will be used to save/retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: `Adaptors` class. :return: `Selectors` class.
""" """
try: try:
elements = self._root.xpath(selector, **kwargs) elements = self._root.xpath(selector, **kwargs)
if elements: if elements:
if auto_save: if auto_save:
if not self.__auto_match_enabled: if not self.__adaptive_enabled:
log.warning( log.warning(
"Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info." "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
) )
else: else:
self.save(elements[0], identifier or selector) self.save(elements[0], identifier or selector)
return self.__handle_elements(elements) return self.__handle_elements(elements)
elif self.__auto_match_enabled: elif self.__adaptive_enabled:
if auto_match: if adaptive:
element_data = self.retrieve(identifier or selector) element_data = self.retrieve(identifier or selector)
if element_data: if element_data:
elements = self.relocate(element_data, percentage) elements = self.relocate(element_data, percentage)
@@ -690,13 +688,13 @@ class Adaptor(SelectorsGeneration):
return self.__handle_elements(elements) return self.__handle_elements(elements)
else: else:
if auto_match: if adaptive:
log.warning( log.warning(
"Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info." "Argument `adaptive` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
) )
elif auto_save: elif auto_save:
log.warning( log.warning(
"Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info." "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
) )
return self.__handle_elements(elements) return self.__handle_elements(elements)
@@ -713,12 +711,12 @@ class Adaptor(SelectorsGeneration):
self, self,
*args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]],
**kwargs: str, **kwargs: str,
) -> "Adaptors": ) -> "Selectors":
"""Find elements by filters of your creations for ease. """Find elements by filters of your creations for ease.
:param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all. :param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
:param kwargs: The attributes you want to filter elements based on it. :param kwargs: The attributes you want to filter elements based on it.
:return: The `Adaptors` object of the elements or empty list :return: The `Selectors` object of the elements or empty list
""" """
# Attributes that are Python reserved words and can't be used directly # Attributes that are Python reserved words and can't be used directly
# Ex: find_all('a', class="blah") -> find_all('a', class_="blah") # Ex: find_all('a', class="blah") -> find_all('a', class_="blah")
@@ -735,7 +733,7 @@ class Adaptor(SelectorsGeneration):
attributes = dict() attributes = dict()
tags, patterns = set(), set() tags, patterns = set(), set()
results, functions, selectors = Adaptors([]), [], [] results, functions, selectors = Selectors([]), [], []
# Brace yourself for a wonderful journey! # Brace yourself for a wonderful journey!
for arg in args: for arg in args:
@@ -766,7 +764,7 @@ class Adaptor(SelectorsGeneration):
functions.append(arg) functions.append(arg)
else: else:
raise TypeError( raise TypeError(
"Callable filter function must have at least one argument to take `Adaptor` objects." "Callable filter function must have at least one argument to take `Selector` objects."
) )
else: else:
@@ -820,12 +818,12 @@ class Adaptor(SelectorsGeneration):
self, self,
*args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]],
**kwargs: str, **kwargs: str,
) -> Union["Adaptor", None]: ) -> Union["Selector", None]:
"""Find elements by filters of your creations for ease, then return the first result. Otherwise return `None`. """Find elements by filters of your creations for ease, then return the first result. Otherwise return `None`.
:param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all. :param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
:param kwargs: The attributes you want to filter elements based on it. :param kwargs: The attributes you want to filter elements based on it.
:return: The `Adaptor` object of the element or `None` if the result didn't match :return: The `Selector` object of the element or `None` if the result didn't match
""" """
for element in self.find_all(*args, **kwargs): for element in self.find_all(*args, **kwargs):
return element return element
@@ -928,15 +926,15 @@ class Adaptor(SelectorsGeneration):
return score return score
def save( def save(
self, element: Union["Adaptor", html.HtmlElement], identifier: str self, element: Union["Selector", html.HtmlElement], identifier: str
) -> None: ) -> None:
"""Saves the element's unique properties to the storage for retrieval and relocation later """Saves the element's unique properties to the storage for retrieval and relocation later
:param element: The element itself that we want to save to storage, it can be an ` Adaptor ` or pure ` HtmlElement ` :param element: The element itself that we want to save to storage, it can be an ` Selector ` or pure ` HtmlElement `
:param identifier: This is the identifier that will be used to retrieve the element later from the storage. See :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
the docs for more info. the docs for more info.
""" """
if self.__auto_match_enabled: if self.__adaptive_enabled:
if isinstance(element, self.__class__): if isinstance(element, self.__class__):
element = element._root element = element._root
@@ -956,7 +954,7 @@ class Adaptor(SelectorsGeneration):
the docs for more info. the docs for more info.
:return: A dictionary of the unique properties :return: A dictionary of the unique properties
""" """
if self.__auto_match_enabled: if self.__adaptive_enabled:
return self._storage.retrieve(identifier) return self._storage.retrieve(identifier)
log.critical( log.critical(
@@ -1065,7 +1063,7 @@ class Adaptor(SelectorsGeneration):
"src", "src",
), ),
match_text: bool = False, match_text: bool = False,
) -> Union["Adaptors[Adaptor]", List]: ) -> Union["Selectors[Selector]", List]:
"""Find elements that are in the same tree depth in the page with the same tag name and same parent tag etc... """Find elements that are in the same tree depth in the page with the same tag name and same parent tag etc...
then return the ones that match the current element attributes with a percentage higher than the input threshold. then return the ones that match the current element attributes with a percentage higher than the input threshold.
@@ -1084,7 +1082,7 @@ class Adaptor(SelectorsGeneration):
:param match_text: If True, element text content will be taken into calculation while matching. :param match_text: If True, element text content will be taken into calculation while matching.
Not recommended to use in normal cases, but it depends. Not recommended to use in normal cases, but it depends.
:return: A ``Adaptors`` container of ``Adaptor`` objects or empty list :return: A ``Selectors`` container of ``Selector`` objects or empty list
""" """
# We will use the elements' root from now on to get the speed boost of using Lxml directly # We will use the elements' root from now on to get the speed boost of using Lxml directly
root = self._root root = self._root
@@ -1128,7 +1126,7 @@ class Adaptor(SelectorsGeneration):
partial: bool = False, partial: bool = False,
case_sensitive: bool = False, case_sensitive: bool = False,
clean_match: bool = True, clean_match: bool = True,
) -> Union["Adaptors[Adaptor]", "Adaptor"]: ) -> Union["Selectors[Selector]", "Selector"]:
"""Find elements that its text content fully/partially matches input. """Find elements that its text content fully/partially matches input.
:param text: Text query to match :param text: Text query to match
:param first_match: Returns the first element that matches conditions, enabled by default :param first_match: Returns the first element that matches conditions, enabled by default
@@ -1137,7 +1135,7 @@ class Adaptor(SelectorsGeneration):
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
""" """
results = Adaptors([]) results = Selectors([])
if not case_sensitive: if not case_sensitive:
text = text.lower() text = text.lower()
@@ -1174,14 +1172,14 @@ class Adaptor(SelectorsGeneration):
first_match: bool = True, first_match: bool = True,
case_sensitive: bool = False, case_sensitive: bool = False,
clean_match: bool = True, clean_match: bool = True,
) -> Union["Adaptors[Adaptor]", "Adaptor"]: ) -> Union["Selectors[Selector]", "Selector"]:
"""Find elements that its text content matches the input regex pattern. """Find elements that its text content matches the input regex pattern.
:param query: Regex query/pattern to match :param query: Regex query/pattern to match
:param first_match: Return the first element that matches conditions; enabled by default. :param first_match: Return the first element that matches conditions; enabled by default.
:param case_sensitive: If enabled, the letters case will be taken into consideration in the regex. :param case_sensitive: If enabled, the letters case will be taken into consideration in the regex.
:param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching. :param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching.
""" """
results = Adaptors([]) results = Selectors([])
# This selector gets all elements with text content # This selector gets all elements with text content
for node in self.__handle_elements( for node in self.__handle_elements(
@@ -1206,24 +1204,24 @@ class Adaptor(SelectorsGeneration):
return results return results
class Adaptors(List[Adaptor]): class Selectors(List[Selector]):
""" """
The `Adaptors` class is a subclass of the builtin ``List`` class, which provides a few additional methods. The `Selectors` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
""" """
__slots__ = () __slots__ = ()
@typing.overload @typing.overload
def __getitem__(self, pos: SupportsIndex) -> Adaptor: def __getitem__(self, pos: SupportsIndex) -> Selector:
pass pass
@typing.overload @typing.overload
def __getitem__(self, pos: slice) -> "Adaptors": def __getitem__(self, pos: slice) -> "Selectors":
pass pass
def __getitem__( def __getitem__(
self, pos: Union[SupportsIndex, slice] self, pos: Union[SupportsIndex, slice]
) -> Union[Adaptor, "Adaptors"]: ) -> Union[Selector, "Selectors"]:
lst = super().__getitem__(pos) lst = super().__getitem__(pos)
if isinstance(pos, slice): if isinstance(pos, slice):
return self.__class__(lst) return self.__class__(lst)
@@ -1237,10 +1235,10 @@ class Adaptors(List[Adaptor]):
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
**kwargs: Any, **kwargs: Any,
) -> "Adaptors[Adaptor]": ) -> "Selectors[Selector]":
""" """
Call the ``.xpath()`` method for each element in this list and return Call the ``.xpath()`` method for each element in this list and return
their results as another `Adaptors` class. their results as another `Selectors` class.
**Important: **Important:
It's recommended to use the identifier argument if you plan to use a different selector later It's recommended to use the identifier argument if you plan to use a different selector later
@@ -1251,12 +1249,12 @@ class Adaptors(List[Adaptor]):
:param selector: The XPath selector to be used. :param selector: The XPath selector to be used.
:param identifier: A string that will be used to retrieve element's data in auto-matching, :param identifier: A string that will be used to retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: `Adaptors` class. :return: `Selectors` class.
""" """
results = [ results = [
n.xpath( n.xpath(
@@ -1272,10 +1270,10 @@ class Adaptors(List[Adaptor]):
identifier: str = "", identifier: str = "",
auto_save: bool = False, auto_save: bool = False,
percentage: int = 0, percentage: int = 0,
) -> "Adaptors[Adaptor]": ) -> "Selectors[Selector]":
""" """
Call the ``.css()`` method for each element in this list and return Call the ``.css()`` method for each element in this list and return
their results flattened as another `Adaptors` class. their results flattened as another `Selectors` class.
**Important: **Important:
It's recommended to use the identifier argument if you plan to use a different selector later It's recommended to use the identifier argument if you plan to use a different selector later
@@ -1284,12 +1282,12 @@ class Adaptors(List[Adaptor]):
:param selector: The CSS3 selector to be used. :param selector: The CSS3 selector to be used.
:param identifier: A string that will be used to retrieve element's data in auto-matching, :param identifier: A string that will be used to retrieve element's data in auto-matching,
otherwise the selector will be used. otherwise the selector will be used.
:param auto_save: Automatically save new elements for `auto_match` later :param auto_save: Automatically save new elements for `adaptive` later
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure, so don't play with this Be aware that the percentage calculation depends solely on the page structure, so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: `Adaptors` class. :return: `Selectors` class.
""" """
results = [ results = [
n.css(selector, identifier or selector, False, auto_save, percentage) n.css(selector, identifier or selector, False, auto_save, percentage)
@@ -1340,7 +1338,7 @@ class Adaptors(List[Adaptor]):
return result return result
return default return default
def search(self, func: Callable[["Adaptor"], bool]) -> Union["Adaptor", None]: def search(self, func: Callable[["Selector"], bool]) -> Union["Selector", None]:
"""Loop over all current elements and return the first element that matches the passed function """Loop over all current elements and return the first element that matches the passed function
:param func: A function that takes each element as an argument and returns True/False :param func: A function that takes each element as an argument and returns True/False
:return: The first element that match the function or ``None`` otherwise. :return: The first element that match the function or ``None`` otherwise.
@@ -1350,10 +1348,10 @@ class Adaptors(List[Adaptor]):
return element return element
return None return None
def filter(self, func: Callable[["Adaptor"], bool]) -> "Adaptors[Adaptor]": def filter(self, func: Callable[["Selector"], bool]) -> "Selectors[Selector]":
"""Filter current elements based on the passed function """Filter current elements based on the passed function
:param func: A function that takes each element as an argument and returns True/False :param func: A function that takes each element as an argument and returns True/False
:return: The new `Adaptors` object or empty list otherwise. :return: The new `Selectors` object or empty list otherwise.
""" """
return self.__class__([element for element in self if func(element)]) return self.__class__([element for element in self if func(element)])
@@ -1382,4 +1380,4 @@ class Adaptors(List[Adaptor]):
def __getstate__(self) -> Any: def __getstate__(self) -> Any:
# lxml don't like it :) # lxml don't like it :)
raise TypeError("Can't pickle Adaptors object") raise TypeError("Can't pickle Selectors object")
tests/fetchers/async/test_camoufox.py
+1 -1
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling import StealthyFetcher from scrapling import StealthyFetcher
StealthyFetcher.auto_match = True StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/fetchers/async/test_dynamic.py
+1 -1
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling import DynamicFetcher from scrapling import DynamicFetcher
DynamicFetcher.auto_match = True DynamicFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/fetchers/async/test_requests.py
+1 -1
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling.fetchers import AsyncFetcher from scrapling.fetchers import AsyncFetcher
AsyncFetcher.auto_match = True AsyncFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/fetchers/sync/test_camoufox.py
+1 -1
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling import StealthyFetcher from scrapling import StealthyFetcher
StealthyFetcher.auto_match = True StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/fetchers/sync/test_dynamic.py
+1 -1
View File
@@ -5,7 +5,7 @@ import pytest_httpbin
from scrapling import DynamicFetcher from scrapling import DynamicFetcher
DynamicFetcher.auto_match = True DynamicFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/fetchers/sync/test_requests.py
+1 -1
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling import Fetcher from scrapling import Fetcher
Fetcher.auto_match = True Fetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
tests/parser/test_automatch.py → tests/parser/test_adaptive.py
+8 -8
View File
@@ -2,10 +2,10 @@ import asyncio
import pytest import pytest
from scrapling import Adaptor from scrapling import Selector
class TestParserAutoMatch: class TestParserAdaptive:
def test_element_relocation(self): def test_element_relocation(self):
"""Test relocating element after structure change""" """Test relocating element after structure change"""
original_html = """ original_html = """
@@ -43,13 +43,13 @@ class TestParserAutoMatch:
</div> </div>
""" """
old_page = Adaptor(original_html, url="example.com", auto_match=True) old_page = Selector(original_html, url="example.com", adaptive=True)
new_page = Adaptor(changed_html, url="example.com", auto_match=True) new_page = Selector(changed_html, url="example.com", adaptive=True)
# 'p1' was used as ID and now it's not and all the path elements have changes # 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors # Also at the same time testing auto-match vs combined selectors
_ = old_page.css("#p1, #p2", auto_save=True)[0] _ = old_page.css("#p1, #p2", auto_save=True)[0]
relocated = new_page.css("#p1", auto_match=True) relocated = new_page.css("#p1", adaptive=True)
assert relocated is not None assert relocated is not None
assert relocated[0].attrib["data-id"] == "p1" assert relocated[0].attrib["data-id"] == "p1"
@@ -97,13 +97,13 @@ class TestParserAutoMatch:
# Simulate async operation # Simulate async operation
await asyncio.sleep(0.1) # Minimal async operation await asyncio.sleep(0.1) # Minimal async operation
old_page = Adaptor(original_html, url="example.com", auto_match=True) old_page = Selector(original_html, url="example.com", adaptive=True)
new_page = Adaptor(changed_html, url="example.com", auto_match=True) new_page = Selector(changed_html, url="example.com", adaptive=True)
# 'p1' was used as ID and now it's not and all the path elements have changes # 'p1' was used as ID and now it's not and all the path elements have changes
# Also at the same time testing auto-match vs combined selectors # Also at the same time testing auto-match vs combined selectors
_ = old_page.css("#p1, #p2", auto_save=True)[0] _ = old_page.css("#p1, #p2", auto_save=True)[0]
relocated = new_page.css("#p1", auto_match=True) relocated = new_page.css("#p1", adaptive=True)
assert relocated is not None assert relocated is not None
assert relocated[0].attrib["data-id"] == "p1" assert relocated[0].attrib["data-id"] == "p1"
tests/parser/test_general.py
+12 -12
View File
@@ -4,7 +4,7 @@ import time
import pytest import pytest
from cssselect import SelectorError, SelectorSyntaxError from cssselect import SelectorError, SelectorSyntaxError
from scrapling import Adaptor from scrapling import Selector
@pytest.fixture @pytest.fixture
@@ -78,7 +78,7 @@ def html_content():
@pytest.fixture @pytest.fixture
def page(html_content): def page(html_content):
return Adaptor(html_content, auto_match=False) return Selector(html_content, adaptive=False)
# CSS Selector Tests # CSS Selector Tests
@@ -162,26 +162,26 @@ class TestSimilarElements:
# Error Handling Tests # Error Handling Tests
class TestErrorHandling: class TestErrorHandling:
def test_invalid_adaptor_initialization(self): def test_invalid_selector_initialization(self):
"""Test various invalid Adaptor initializations""" """Test various invalid Selector initializations"""
# No arguments # No arguments
with pytest.raises(ValueError): with pytest.raises(ValueError):
_ = Adaptor(auto_match=False) _ = Selector(adaptive=False)
# Invalid argument types # Invalid argument types
with pytest.raises(TypeError): with pytest.raises(TypeError):
_ = Adaptor(root="ayo", auto_match=False) _ = Selector(root="ayo", adaptive=False)
with pytest.raises(TypeError): with pytest.raises(TypeError):
_ = Adaptor(text=1, auto_match=False) _ = Selector(text=1, adaptive=False)
with pytest.raises(TypeError): with pytest.raises(TypeError):
_ = Adaptor(body=1, auto_match=False) _ = Selector(body=1, adaptive=False)
def test_invalid_storage(self, page, html_content): def test_invalid_storage(self, page, html_content):
"""Test invalid storage parameter""" """Test invalid storage parameter"""
with pytest.raises(ValueError): with pytest.raises(ValueError):
_ = Adaptor(html_content, storage=object, auto_match=True) _ = Selector(html_content, storage=object, adaptive=True)
def test_bad_selectors(self, page): def test_bad_selectors(self, page):
"""Test handling of invalid selectors""" """Test handling of invalid selectors"""
@@ -195,7 +195,7 @@ class TestErrorHandling:
# Pickling and Object Representation Tests # Pickling and Object Representation Tests
class TestPicklingAndRepresentation: class TestPicklingAndRepresentation:
def test_unpickleable_objects(self, page): def test_unpickleable_objects(self, page):
"""Test that Adaptor objects cannot be pickled""" """Test that Selector objects cannot be pickled"""
table = page.css(".product-list")[0] table = page.css(".product-list")[0]
with pytest.raises(TypeError): with pytest.raises(TypeError):
pickle.dumps(table) pickle.dumps(table)
@@ -299,7 +299,7 @@ def test_large_html_parsing_performance():
) )
start_time = time.time() start_time = time.time()
parsed = Adaptor(large_html, auto_match=False) parsed = Selector(large_html, adaptive=False)
elements = parsed.css(".item") elements = parsed.css(".item")
end_time = time.time() end_time = time.time()
@@ -315,7 +315,7 @@ def test_large_html_parsing_performance():
def test_selectors_generation(page): def test_selectors_generation(page):
"""Try to create selectors for all elements in the page""" """Try to create selectors for all elements in the page"""
def _traverse(element: Adaptor): def _traverse(element: Selector):
assert isinstance(element.generate_css_selector, str) assert isinstance(element.generate_css_selector, str)
assert isinstance(element.generate_xpath_selector, str) assert isinstance(element.generate_xpath_selector, str)
for branch in element.children: for branch in element.children: