freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tests: Update test to be up-to-date with current version of the code

Browse Source
This commit is contained in:
Karim shoair
2025-09-14 16:30:59 +03:00
parent 0b3509a295
commit 1024ba6916
1 changed files with 91 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/cli/test_cli.py
+91 -91
View File
@@ -3,12 +3,23 @@ from click.testing import CliRunner
from unittest.mock import patch, MagicMock from unittest.mock import patch, MagicMock
import pytest_httpbin import pytest_httpbin
from scrapling.parser import Selector
from scrapling.cli import ( from scrapling.cli import (
shell, mcp, get, post, put, delete, fetch, stealthy_fetch shell, mcp, get, post, put, delete, fetch, stealthy_fetch
) )
@pytest_httpbin.use_class_based_httpbin @pytest_httpbin.use_class_based_httpbin
def configure_selector_mock():
"""Helper function to create a properly configured Selector mock"""
mock_response = MagicMock(spec=Selector)
mock_response.body = "<html><body>Test content</body></html>"
mock_response.get_all_text.return_value = "Test content"
mock_response.css_first.return_value = mock_response
mock_response.css.return_value = [mock_response]
return mock_response
class TestCLI: class TestCLI:
"""Test CLI functionality""" """Test CLI functionality"""
@@ -45,136 +56,129 @@ class TestCLI:
output_file = tmp_path / "output.md" output_file = tmp_path / "output.md"
with patch('scrapling.fetchers.Fetcher.get') as mock_get: with patch('scrapling.fetchers.Fetcher.get') as mock_get:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_response.status = 200 mock_response.status = 200
mock_get.return_value = mock_response mock_get.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( get,
get, [html_url, str(output_file)]
[html_url, str(output_file)] )
) assert result.exit_code == 0
assert result.exit_code == 0
# Test with various options # Test with various options
with patch('scrapling.fetchers.Fetcher.get') as mock_get: with patch('scrapling.fetchers.Fetcher.get') as mock_get:
mock_get.return_value = mock_response mock_get.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( get,
get, [
[ html_url,
html_url, str(output_file),
str(output_file), '-H', 'User-Agent: Test',
'-H', 'User-Agent: Test', '--cookies', 'session=abc123',
'--cookies', 'session=abc123', '--timeout', '60',
'--timeout', '60', '--proxy', 'http://proxy:8080',
'--proxy', 'http://proxy:8080', '-s', '.content',
'-s', '.content', '-p', 'page=1'
'-p', 'page=1' ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_extract_post_command(self, runner, tmp_path, html_url): def test_extract_post_command(self, runner, tmp_path, html_url):
"""Test extract `post` command""" """Test extract `post` command"""
output_file = tmp_path / "output.html" output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.post') as mock_post: with patch('scrapling.fetchers.Fetcher.post') as mock_post:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_post.return_value = mock_response mock_post.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( post,
post, [
[ html_url,
html_url, str(output_file),
str(output_file), '-d', 'key=value',
'-d', 'key=value', '-j', '{"data": "test"}'
'-j', '{"data": "test"}' ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_extract_put_command(self, runner, tmp_path, html_url): def test_extract_put_command(self, runner, tmp_path, html_url):
"""Test extract `put` command""" """Test extract `put` command"""
output_file = tmp_path / "output.html" output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.put') as mock_put: with patch('scrapling.fetchers.Fetcher.put') as mock_put:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_put.return_value = mock_response mock_put.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( put,
put, [
[ html_url,
html_url, str(output_file),
str(output_file), '-d', 'key=value',
'-d', 'key=value', '-j', '{"data": "test"}'
'-j', '{"data": "test"}' ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_extract_delete_command(self, runner, tmp_path, html_url): def test_extract_delete_command(self, runner, tmp_path, html_url):
"""Test extract `delete` command""" """Test extract `delete` command"""
output_file = tmp_path / "output.html" output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.delete') as mock_delete: with patch('scrapling.fetchers.Fetcher.delete') as mock_delete:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_delete.return_value = mock_response mock_delete.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( delete,
delete, [
[ html_url,
html_url, str(output_file)
str(output_file) ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_extract_fetch_command(self, runner, tmp_path, html_url): def test_extract_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command""" """Test extract fetch command"""
output_file = tmp_path / "output.txt" output_file = tmp_path / "output.txt"
with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch: with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_fetch.return_value = mock_response mock_fetch.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( fetch,
fetch, [
[ html_url,
html_url, str(output_file),
str(output_file), '--headless',
'--headless', '--stealth',
'--stealth', '--timeout', '60000'
'--timeout', '60000' ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url): def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command""" """Test extract fetch command"""
output_file = tmp_path / "output.md" output_file = tmp_path / "output.md"
with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch: with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch:
mock_response = MagicMock() mock_response = configure_selector_mock()
mock_fetch.return_value = mock_response mock_fetch.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'): result = runner.invoke(
result = runner.invoke( stealthy_fetch,
stealthy_fetch, [
[ html_url,
html_url, str(output_file),
str(output_file), '--headless',
'--headless', '--css-selector', 'body',
'--css-selector', 'body', '--timeout', '60000'
'--timeout', '60000' ]
] )
) assert result.exit_code == 0
assert result.exit_code == 0
def test_invalid_arguments(self, runner, html_url): def test_invalid_arguments(self, runner, html_url):
"""Test invalid arguments handling""" """Test invalid arguments handling"""
@@ -182,12 +186,8 @@ class TestCLI:
result = runner.invoke(get) result = runner.invoke(get)
assert result.exit_code != 0 assert result.exit_code != 0
# Invalid output file extension _ = runner.invoke(
with patch('scrapling.cli.Convertor.write_content_to_file') as mock_write: get,
mock_write.side_effect = ValueError("Unknown file type") [html_url, 'output.invalid']
)
_ = runner.invoke( # Should handle the error gracefully
get,
[html_url, 'output.invalid']
)
# Should handle the error gracefully