feat(spiders): add simple export system for the results

scrapling/spiders/engine.py

@@ -5,7 +5,7 @@ from anyio import create_task_group, CapacityLimiter
 
 from scrapling.core.utils import log
 from scrapling.spiders.request import Request
-from scrapling.spiders.result import CrawlStats
+from scrapling.spiders.result import CrawlStats, ItemList
 from scrapling.spiders.scheduler import Scheduler
 from scrapling.spiders.session import SessionManager
 from scrapling.core._types import Dict, TYPE_CHECKING, Any
@@ -33,7 +33,7 @@ class CrawlerEngine:
 
         self._active_tasks: int = 0
         self._running: bool = False
-        self._items: list[dict[str, Any]] = []
+        self._items: ItemList = ItemList()
 
     def _is_domain_allowed(self, request: Request) -> bool:
         """Check if the request's domain is in allowed_domains."""
@@ -161,6 +161,6 @@ class CrawlerEngine:
         return self.stats
 
     @property
-    def items(self) -> list[dict[str, Any]]:
+    def items(self) -> ItemList:
         """Access scraped items."""
         return self._items

scrapling/spiders/result.py

@@ -1,6 +1,42 @@
+from pathlib import Path
 from dataclasses import dataclass, field
 
-from scrapling.core._types import Any, Iterator, Dict, List, Tuple
+import orjson
+
+from scrapling.core.utils import log
+from scrapling.core._types import Any, Iterator, Dict, List, Tuple, Union
+
+
+class ItemList(list):
+    """A list of scraped items with export capabilities."""
+
+    def to_json(self, path: Union[str, Path], *, indent: bool = False):
+        """Export items to a JSON file.
+
+        :param path: Path to the output file
+        :param indent: Pretty-print with 2-space indentation (slightly slower)
+        :return: Number of items written
+        """
+        options = orjson.OPT_SERIALIZE_NUMPY
+        if indent:
+            options |= orjson.OPT_INDENT_2
+
+        file = Path(path)
+        file.parent.mkdir(parents=True, exist_ok=True)
+        file.write_bytes(orjson.dumps(list(self), option=options))
+        log.info("Saved %d items to %s", len(self), path)
+
+    def to_jsonl(self, path: Union[str, Path]):
+        """Export items as JSON Lines (one JSON object per line).
+
+        :param path: Path to the output file
+        """
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "wb") as f:
+            for item in self:
+                f.write(orjson.dumps(item, option=orjson.OPT_SERIALIZE_NUMPY))
+                f.write(b"\n")
+        log.info("Saved %d items to %s", len(self), path)
 
 
 @dataclass
@@ -73,7 +109,7 @@ class CrawlResult:
     """Complete result from a spider run."""
 
     stats: CrawlStats
-    items: list[dict[str, Any]]
+    items: ItemList
 
     def __len__(self) -> int:
         return len(self.items)