fix(defaults): Lazy Loading fetchers to lower memory usage on importing
This commit is contained in:
+18
-9
@@ -1,10 +1,19 @@
|
||||
from .fetchers import AsyncFetcher as _AsyncFetcher
|
||||
from .fetchers import Fetcher as _Fetcher
|
||||
from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
|
||||
from .fetchers import StealthyFetcher as _StealthyFetcher
|
||||
|
||||
# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
|
||||
Fetcher = _Fetcher()
|
||||
AsyncFetcher = _AsyncFetcher()
|
||||
StealthyFetcher = _StealthyFetcher()
|
||||
PlayWrightFetcher = _PlayWrightFetcher()
|
||||
|
||||
# A lightweight approach to create lazy loader for each import for backward compatibility
|
||||
# This will reduces initial memory footprint significantly (only loads what's used)
|
||||
def __getattr__(name):
|
||||
if name == 'Fetcher':
|
||||
from scrapling.fetchers import Fetcher as cls
|
||||
return cls()
|
||||
elif name == 'AsyncFetcher':
|
||||
from scrapling.fetchers import AsyncFetcher as cls
|
||||
return cls()
|
||||
elif name == 'StealthyFetcher':
|
||||
from scrapling.fetchers import StealthyFetcher as cls
|
||||
return cls()
|
||||
elif name == 'PlayWrightFetcher':
|
||||
from scrapling.fetchers import PlayWrightFetcher as cls
|
||||
return cls()
|
||||
else:
|
||||
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
|
||||
|
||||
Reference in New Issue
Block a user