Merge branch 'dev' into test/edge-cases-filter-ancestors-find-similar

tests/parser/test_parser_advanced.py

@@ -250,6 +250,68 @@ class TestTextHandlerAdvanced:
         matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False)
         assert len(matches) == 1
 
+    def test_text_handler_regex_check_match(self):
+        """Test TextHandler.re() with check_match=True returns bool"""
+        text = TextHandler("Price: $10.99")
+        assert text.re(r"\$[\d.]+", check_match=True) is True
+        assert text.re(r"no-match-pattern", check_match=True) is False
+
+    def test_text_handler_regex_replace_entities_false(self):
+        """Test TextHandler.re() with replace_entities=False preserves entities"""
+        text = TextHandler("Hello & World")
+        results = text.re(r"&", replace_entities=False)
+        assert len(results) == 1
+        assert results[0] == "&"
+
+    def test_text_handler_regex_with_groups(self):
+        """Test TextHandler.re() with capture groups flattens results"""
+        text = TextHandler("name=Alice age=30 name=Bob age=25")
+        results = text.re(r"name=(\w+) age=(\d+)")
+        assert len(results) == 4
+        assert "Alice" in results
+        assert "30" in results
+
+    def test_text_handler_re_first_with_default(self):
+        """Test TextHandler.re_first() returns default when no match"""
+        text = TextHandler("no numbers here")
+        result = text.re_first(r"\d+", default="N/A")
+        assert result == "N/A"
+
+    def test_text_handler_re_first_returns_first_match(self):
+        """Test TextHandler.re_first() returns first match"""
+        text = TextHandler("a1 b2 c3")
+        result = text.re_first(r"\d")
+        assert result == "1"
+        assert isinstance(result, TextHandler)
+
+    def test_text_handler_clean_with_entities(self):
+        """Test TextHandler.clean() with remove_entities=True"""
+        text = TextHandler("Hello\t&\nWorld")
+        cleaned = text.clean(remove_entities=True)
+        assert "&" not in cleaned
+        assert "&" in cleaned
+        assert "\t" not in cleaned
+        assert "\n" not in cleaned
+
+    def test_text_handler_clean_without_entities(self):
+        """Test TextHandler.clean() preserves entities by default"""
+        text = TextHandler("Hello\t&\nWorld")
+        cleaned = text.clean(remove_entities=False)
+        assert "&" in cleaned
+
+    def test_text_handler_json_valid(self):
+        """Test TextHandler.json() with valid JSON"""
+        text = TextHandler('{"key": "value", "num": 42}')
+        data = text.json()
+        assert data["key"] == "value"
+        assert data["num"] == 42
+
+    def test_text_handler_json_invalid(self):
+        """Test TextHandler.json() raises on invalid JSON"""
+        text = TextHandler("not json")
+        with pytest.raises(Exception):
+            text.json()
+
     def test_text_handlers_operations(self):
         """Test TextHandlers list operations"""
         handlers = TextHandlers([
@@ -266,6 +328,37 @@ class TestTextHandlerAdvanced:
         assert handlers.get("default") == "First"
         assert TextHandlers([]).get("default") == "default"
 
+    def test_text_handlers_re(self):
+        """Test TextHandlers.re() flattens results across all elements"""
+        handlers = TextHandlers([
+            TextHandler("a1 b2"),
+            TextHandler("c3 d4"),
+        ])
+        results = handlers.re(r"[a-z]\d")
+        assert isinstance(results, TextHandlers)
+        assert len(results) == 4
+        assert results[0] == "a1"
+        assert results[3] == "d4"
+
+    def test_text_handlers_re_empty(self):
+        """Test TextHandlers.re() on empty list"""
+        handlers = TextHandlers([])
+        results = handlers.re(r"\d+")
+        assert isinstance(results, TextHandlers)
+        assert len(results) == 0
+
+    def test_text_handlers_re_no_matches(self):
+        """Test TextHandlers.re() when no element matches"""
+        handlers = TextHandlers([TextHandler("abc"), TextHandler("def")])
+        results = handlers.re(r"\d+")
+        assert len(results) == 0
+
+    def test_text_handlers_extract(self):
+        """Test TextHandlers.extract() returns self"""
+        handlers = TextHandlers([TextHandler("a"), TextHandler("b")])
+        assert handlers.extract() is handlers
+        assert handlers.getall() is handlers
+
 
 class TestSelectorsAdvanced:
     """Test advanced Selectors functionality"""

tests/spiders/test_engine.py

@@ -613,8 +613,7 @@ class TestCheckpointMethods:
     @pytest.mark.asyncio
     async def test_restore_from_checkpoint_raises_when_disabled(self):
         engine = _make_engine()  # no crawldir → checkpoint disabled
-        with pytest.raises(RuntimeError):
-            await engine._restore_from_checkpoint()
+        assert (await engine._restore_from_checkpoint()) is False
 
 
 # ---------------------------------------------------------------------------

tests/spiders/test_session.py

@@ -1,9 +1,12 @@
 """Tests for the SessionManager class."""
 
+from unittest.mock import AsyncMock, PropertyMock
+
 from scrapling.core._types import Any
 import pytest
 
 from scrapling.spiders.session import SessionManager
+from scrapling.spiders.request import Request
 
 
 class MockSession:  # type: ignore[type-arg]
@@ -350,3 +353,57 @@ class TestSessionManagerIntegration:
         # After close - all inactive
         await manager.close()
         assert all(not s._is_alive for s in sessions)
+
+
+class TestSessionManagerFetch:
+    """Test SessionManager fetch behavior."""
+
+    @pytest.mark.asyncio
+    async def test_fetch_preserves_request_method(self):
+        """Test that fetch does not mutate request._session_kwargs.
+
+        Previously, fetch() used pop("method") which removed the method
+        key from the original request dict. This caused retried requests
+        (via request.copy()) to lose their HTTP method and fall back to GET.
+        """
+        from scrapling.engines.static import _ASyncSessionLogic
+        from scrapling.fetchers import FetcherSession
+        from scrapling.engines.toolbelt.custom import Response
+
+        mock_response = Response(
+            url="https://example.com",
+            content=b"ok",
+            status=200,
+            reason="OK",
+            cookies={},
+            headers={"content-type": "text/html"},
+            request_headers={},
+        )
+        mock_response.meta = {}
+
+        mock_client = AsyncMock(spec=_ASyncSessionLogic)
+        mock_client._make_request = AsyncMock(return_value=mock_response)
+
+        mock_session = AsyncMock(spec=FetcherSession)
+        mock_session._client = mock_client
+        mock_session._is_alive = True
+
+        manager = SessionManager()
+        manager._sessions["default"] = mock_session
+        manager._default_session_id = "default"
+        manager._started = True
+
+        request = Request("https://example.com", method="POST", data={"key": "value"})
+
+        assert request._session_kwargs["method"] == "POST"
+
+        await manager.fetch(request)
+
+        # method must still be present after fetch
+        assert "method" in request._session_kwargs
+        assert request._session_kwargs["method"] == "POST"
+
+        # verify the correct method was passed to _make_request
+        mock_client._make_request.assert_called_once()
+        call_kwargs = mock_client._make_request.call_args
+        assert call_kwargs.kwargs["method"] == "POST"