From d0b2cec849913a406d1f31ed6c3e52357e8bf32c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 12 Feb 2026 03:30:14 +0200 Subject: [PATCH] docs: simplify the architecture mermaid --- docs/spiders/architecture.md | 58 ++++++++---------------------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/docs/spiders/architecture.md b/docs/spiders/architecture.md index 70fba8c..412243b 100644 --- a/docs/spiders/architecture.md +++ b/docs/spiders/architecture.md @@ -15,57 +15,23 @@ The diagram below shows how data flows through the spider system when a crawl is ```mermaid graph TB - subgraph Spider["🕷️ Spider"] - direction TB - SR["start_requests()"] - Parse["parse() / callbacks"] - Hooks["Lifecycle hooks
on_start · on_close · on_error
is_blocked · on_scraped_item
"] - end + Spider["Spider"] + Scheduler["Scheduler"] + Engine["Crawler Engine"] + SessionMgr["Session Manager"] + Output["Output"] + Checkpoint["Checkpoint"] - subgraph Engine["⚙️ Crawler Engine"] - direction TB - Loop["Crawl loop"] - Conc["Concurrency control
Global limiter · Per-domain limiter
Download delay
"] - BlockDet["Blocked request
detection & retry"] - end - - subgraph Scheduler["📋 Scheduler"] - PQ["Priority queue"] - Dedup["Fingerprint
deduplication"] - end - - subgraph SessionMgr["🔌 Session Manager"] - direction TB - Router["Request router
Routes by session ID"] - subgraph Sessions["Sessions"] - direction LR - HTTP["FetcherSession
HTTP requests"] - Dynamic["AsyncDynamicSession
Browser automation"] - Stealth["AsyncStealthySession
Anti-bot bypass"] - end - end - - subgraph Output["📦 Output"] - Items["ItemList
to_json() · to_jsonl()"] - Stream["Streaming
async for item in spider.stream()"] - Stats["CrawlStats"] - end - - Checkpoint["💾 Checkpoint
Pause/Resume"] - - %% Data flow - SR -- "1. Initial
Requests" --> Scheduler - Scheduler -- "2. Next
Request" --> Engine + Spider -- "1. Initial Requests" --> Scheduler + Scheduler -- "2. Next Request" --> Engine Engine -- "3. Fetch" --> SessionMgr - Router --> Sessions SessionMgr -- "4. Response" --> Engine - Engine -- "5. Response" --> Parse - Parse -- "6a. New Requests" --> Scheduler - Parse -- "6b. Items (dict)" --> Output - Engine -. "Periodic save" .-> Checkpoint + Engine -- "5. Response to callback" --> Spider + Spider -- "6. New Requests" --> Scheduler + Spider -- "7. Items" --> Output + Engine -. "Save" .-> Checkpoint Checkpoint -. "Resume" .-> Scheduler - %% Styling classDef spiderClass fill:#7c4dff,stroke:#333,color:#fff classDef engineClass fill:#00897b,stroke:#333,color:#fff classDef schedulerClass fill:#1565c0,stroke:#333,color:#fff