fix(defaults): Lazy Loading fetchers to lower memory usage on importing

This commit is contained in:
Karim shoair
2025-03-17 06:56:34 +02:00
parent 94f16ca1af
commit 9b0515b1cd
+18 -9
View File
@@ -1,10 +1,19 @@
from .fetchers import AsyncFetcher as _AsyncFetcher
from .fetchers import Fetcher as _Fetcher
from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
from .fetchers import StealthyFetcher as _StealthyFetcher
# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
Fetcher = _Fetcher()
AsyncFetcher = _AsyncFetcher()
StealthyFetcher = _StealthyFetcher()
PlayWrightFetcher = _PlayWrightFetcher()
# A lightweight approach to create lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name):
if name == 'Fetcher':
from scrapling.fetchers import Fetcher as cls
return cls()
elif name == 'AsyncFetcher':
from scrapling.fetchers import AsyncFetcher as cls
return cls()
elif name == 'StealthyFetcher':
from scrapling.fetchers import StealthyFetcher as cls
return cls()
elif name == 'PlayWrightFetcher':
from scrapling.fetchers import PlayWrightFetcher as cls
return cls()
else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")